From 18e59f34902928e7e6aaa2d194f5187ae10b9556 Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Wed, 19 Apr 2023 12:34:46 +0200 Subject: [PATCH 1/7] ADD Parallel download Tests --- .gitignore | 1 + Tests/BasicTests/ParallelTests.cs | 84 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 Tests/BasicTests/ParallelTests.cs diff --git a/.gitignore b/.gitignore index 63bc327..c134fee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vs obj bin +.vscode \ No newline at end of file diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs new file mode 100644 index 0000000..e0f48de --- /dev/null +++ b/Tests/BasicTests/ParallelTests.cs @@ -0,0 +1,84 @@ +using CodexDistTestCore; +using CodexDistTestCore.Config; +using NUnit.Framework; + +namespace Tests.ParallelTests +{ + [TestFixture] + public class DownloadTests : DistTest + { + [Test] + public void TwoNodeDownloads() + { + ParallelDownload(2, 64.MB()); + } + [Test] + public void FiveNodeDownloads() + { + ParallelDownload(5, 1000.MB()); + } + [Test] + public void TenNodeDownloads() + { + ParallelDownload(10, 16.MB()); + } + public void download(ContentId contentId, CodexDistTestCore.IOnlineCodexNode node, TestFile testFile) + { + var downloadedFile = node.DownloadContent(contentId); + testFile.AssertIsEqual(downloadedFile); + } + + void ParallelDownload(int numberOfNodes, ByteSize filesize) + { + var group = SetupCodexNodes(numberOfNodes).EnableMetrics().BringOnline(); + + var host = group[0]; + + for (int i = 1; i < numberOfNodes; i++) + { + host.ConnectToPeer(group[i]); + } + + var testFile = GenerateTestFile(filesize); + + var contentId = host.UploadFile(testFile); + + for (int i = 1; i < numberOfNodes; i++) + { + new Task(() => { download(contentId, group[i], testFile); }).Start(); + } + Task.WaitAll(); + } + } + + [TestFixture] + public class UploadTests : DistTest + { + [Test] + public void TwoNodeUploads() + { + } + + public void FiveNodeUploads() + { + } + public void TenNodeUploads() + { + } + } + [TestFixture] + public class MixedTests : DistTest + { + [Test] + public void OneDownloadOneUpload() + { + } + + public void ThreeDownloadTwoUpload() + { + } + public void FiveDownloadFiveUpload() + { + } + } +} \ No newline at end of file From a7e024002fe5a2c7db50fc05e23cb6e2040e599a Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Thu, 20 Apr 2023 12:43:25 +0200 Subject: [PATCH 2/7] ADD upload tests --- Tests/BasicTests/ParallelTests.cs | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs index e0f48de..c80eba2 100644 --- a/Tests/BasicTests/ParallelTests.cs +++ b/Tests/BasicTests/ParallelTests.cs @@ -57,13 +57,51 @@ namespace Tests.ParallelTests [Test] public void TwoNodeUploads() { + ParallelUpload(2, 64.MB()); } - + [Test] public void FiveNodeUploads() { + ParallelUpload(5, 1000.MB()); } + [Test] public void TenNodeUploads() { + ParallelUpload(10, 16.MB()); + } + void ParallelUpload(int numberOfNodes, ByteSize filesize) + { + var group = SetupCodexNodes(numberOfNodes).EnableMetrics().BringOnline(); + + var host = group[0]; + + for (int i = 1; i < numberOfNodes; i++) + { + host.ConnectToPeer(group[i]); + } + var testfiles = new List(); + var contentIds = new List(); + for (int i = 1; i < numberOfNodes; i++) + { + testfiles.Add(GenerateTestFile(filesize)); + new Task(() => { upload(host, testfiles[i - 1], contentIds, i - 1); }).Start(); + } + Task.WaitAll(); + for (int i = 0; i < testfiles.Count; i++) + { + new Task(() => { download(contentIds[i], group[i + 1], testfiles[i]); }).Start(); + } + Task.WaitAll(); + } + + void download(ContentId contentId, CodexDistTestCore.IOnlineCodexNode node, TestFile testFile) + { + var downloadedFile = node.DownloadContent(contentId); + testFile.AssertIsEqual(downloadedFile); + } + void upload(CodexDistTestCore.IOnlineCodexNode host, TestFile testfile, List contentIds, int pos) + { + contentIds[pos] = host.UploadFile(testfile); } } [TestFixture] From 71b061cac79d1a7b4c58b15857ef25ecdf0cb3ec Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Thu, 20 Apr 2023 14:16:04 +0200 Subject: [PATCH 3/7] MDF threaded functions not running correctly --- Tests/BasicTests/ParallelTests.cs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs index c80eba2..c9c13b9 100644 --- a/Tests/BasicTests/ParallelTests.cs +++ b/Tests/BasicTests/ParallelTests.cs @@ -1,7 +1,6 @@ using CodexDistTestCore; using CodexDistTestCore.Config; using NUnit.Framework; - namespace Tests.ParallelTests { [TestFixture] @@ -10,15 +9,15 @@ namespace Tests.ParallelTests [Test] public void TwoNodeDownloads() { - ParallelDownload(2, 64.MB()); + ParallelDownload(3, 64.MB()); } [Test] - public void FiveNodeDownloads() + public void FourNodeDownloads() { ParallelDownload(5, 1000.MB()); } [Test] - public void TenNodeDownloads() + public void NineNodeDownloads() { ParallelDownload(10, 16.MB()); } @@ -45,9 +44,10 @@ namespace Tests.ParallelTests for (int i = 1; i < numberOfNodes; i++) { - new Task(() => { download(contentId, group[i], testFile); }).Start(); + // new Task(() => { download(contentId, group[i], testFile); }).Start(); + download(contentId, group[i], testFile); } - Task.WaitAll(); + // Task.WaitAll(); } } @@ -55,9 +55,9 @@ namespace Tests.ParallelTests public class UploadTests : DistTest { [Test] - public void TwoNodeUploads() + public void ThreeNodeUploads() { - ParallelUpload(2, 64.MB()); + ParallelUpload(3, 64.MB()); } [Test] public void FiveNodeUploads() @@ -84,14 +84,16 @@ namespace Tests.ParallelTests for (int i = 1; i < numberOfNodes; i++) { testfiles.Add(GenerateTestFile(filesize)); - new Task(() => { upload(host, testfiles[i - 1], contentIds, i - 1); }).Start(); + // new Task(() => { upload(host, testfiles[i - 1], contentIds, i - 1); }).Start(); + upload(host, testfiles[i - 1], contentIds, i - 1); } - Task.WaitAll(); + // Task.WaitAll(); for (int i = 0; i < testfiles.Count; i++) { - new Task(() => { download(contentIds[i], group[i + 1], testfiles[i]); }).Start(); + // new Task(() => { download(contentIds[i], group[i + 1], testfiles[i]); }).Start(); + download(contentIds[i], group[i + 1], testfiles[i]); } - Task.WaitAll(); + // Task.WaitAll(); } void download(ContentId contentId, CodexDistTestCore.IOnlineCodexNode node, TestFile testFile) From b70e8033f3649f355874cabe5a2f20420a2cc4ec Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Thu, 20 Apr 2023 15:58:47 +0200 Subject: [PATCH 4/7] WIP merge with main --- Tests/BasicTests/ParallelTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs index c9c13b9..138dc41 100644 --- a/Tests/BasicTests/ParallelTests.cs +++ b/Tests/BasicTests/ParallelTests.cs @@ -1,5 +1,5 @@ -using CodexDistTestCore; -using CodexDistTestCore.Config; +using DistTestCore; +using KubernetesWorkflow; using NUnit.Framework; namespace Tests.ParallelTests { @@ -21,7 +21,7 @@ namespace Tests.ParallelTests { ParallelDownload(10, 16.MB()); } - public void download(ContentId contentId, CodexDistTestCore.IOnlineCodexNode node, TestFile testFile) + public void download(ContentId contentId, IOnlineCodexNode node, TestFile testFile) { var downloadedFile = node.DownloadContent(contentId); testFile.AssertIsEqual(downloadedFile); @@ -96,12 +96,12 @@ namespace Tests.ParallelTests // Task.WaitAll(); } - void download(ContentId contentId, CodexDistTestCore.IOnlineCodexNode node, TestFile testFile) + void download(ContentId contentId, IOnlineCodexNode node, TestFile testFile) { var downloadedFile = node.DownloadContent(contentId); testFile.AssertIsEqual(downloadedFile); } - void upload(CodexDistTestCore.IOnlineCodexNode host, TestFile testfile, List contentIds, int pos) + void upload(IOnlineCodexNode host, TestFile testfile, List contentIds, int pos) { contentIds[pos] = host.UploadFile(testfile); } From 5d5b08e084a9a47dcc8d5fa0a8c771395eda9435 Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Tue, 25 Apr 2023 13:43:51 +0200 Subject: [PATCH 5/7] ADD multithreading --- Logging/LogFile.cs | 6 ++- Tests/BasicTests/ParallelTests.cs | 89 ++++++++++++++----------------- 2 files changed, 46 insertions(+), 49 deletions(-) diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index 96ed16b..349e7c6 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -3,6 +3,7 @@ public class LogFile { private readonly string extension; + private readonly object fileLock = new object(); private string filename; public LogFile(string filename, string extension) @@ -25,7 +26,10 @@ { try { - File.AppendAllLines(FullFilename, new[] { message }); + lock (fileLock) + { + File.AppendAllLines(FullFilename, new[] { message }); + } } catch (Exception ex) { diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs index 138dc41..b1701bc 100644 --- a/Tests/BasicTests/ParallelTests.cs +++ b/Tests/BasicTests/ParallelTests.cs @@ -7,47 +7,45 @@ namespace Tests.ParallelTests public class DownloadTests : DistTest { [Test] - public void TwoNodeDownloads() + public void ThreeNodeDownloads() { - ParallelDownload(3, 64.MB()); + ParallelDownload(3, 5000.MB()); } [Test] - public void FourNodeDownloads() + public void FiveNodeDownloads() { ParallelDownload(5, 1000.MB()); } [Test] - public void NineNodeDownloads() + public void TenNodeDownloads() { - ParallelDownload(10, 16.MB()); - } - public void download(ContentId contentId, IOnlineCodexNode node, TestFile testFile) - { - var downloadedFile = node.DownloadContent(contentId); - testFile.AssertIsEqual(downloadedFile); + ParallelDownload(10, 256.MB()); } void ParallelDownload(int numberOfNodes, ByteSize filesize) { - var group = SetupCodexNodes(numberOfNodes).EnableMetrics().BringOnline(); + var group = SetupCodexNodes(numberOfNodes).BringOnline(); + var host = SetupCodexNodes(1).BringOnline()[0]; - var host = group[0]; - - for (int i = 1; i < numberOfNodes; i++) + foreach (var node in group) { - host.ConnectToPeer(group[i]); + host.ConnectToPeer(node); } var testFile = GenerateTestFile(filesize); - var contentId = host.UploadFile(testFile); - - for (int i = 1; i < numberOfNodes; i++) + var list = new List>(); + + foreach (var node in group) { - // new Task(() => { download(contentId, group[i], testFile); }).Start(); - download(contentId, group[i], testFile); + list.Add(Task.Run(() => { return node.DownloadContent(contentId); })); + } + + Task.WaitAll(list.ToArray()); + foreach (var task in list) + { + testFile.AssertIsEqual(task.Result); } - // Task.WaitAll(); } } @@ -57,53 +55,48 @@ namespace Tests.ParallelTests [Test] public void ThreeNodeUploads() { - ParallelUpload(3, 64.MB()); + ParallelUpload(3, 50.MB()); } [Test] public void FiveNodeUploads() { - ParallelUpload(5, 1000.MB()); + ParallelUpload(5, 750.MB()); } [Test] public void TenNodeUploads() { - ParallelUpload(10, 16.MB()); + ParallelUpload(10, 25.MB()); } void ParallelUpload(int numberOfNodes, ByteSize filesize) { - var group = SetupCodexNodes(numberOfNodes).EnableMetrics().BringOnline(); + var group = SetupCodexNodes(numberOfNodes).BringOnline(); + var host = SetupCodexNodes(1).BringOnline()[0]; - var host = group[0]; - - for (int i = 1; i < numberOfNodes; i++) + foreach (var node in group) { - host.ConnectToPeer(group[i]); + host.ConnectToPeer(node); } + var testfiles = new List(); - var contentIds = new List(); - for (int i = 1; i < numberOfNodes; i++) + var contentIds = new List>(); + + for (int i = 0; i < group.Count(); i++) { testfiles.Add(GenerateTestFile(filesize)); - // new Task(() => { upload(host, testfiles[i - 1], contentIds, i - 1); }).Start(); - upload(host, testfiles[i - 1], contentIds, i - 1); + var n = i; + contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); })); } - // Task.WaitAll(); - for (int i = 0; i < testfiles.Count; i++) + var downloads = new List>(); + for (int i = 0; i < group.Count(); i++) { - // new Task(() => { download(contentIds[i], group[i + 1], testfiles[i]); }).Start(); - download(contentIds[i], group[i + 1], testfiles[i]); + var n = i; + downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); })); + } + Task.WaitAll(downloads.ToArray()); + for (int i = 0; i < group.Count(); i++) + { + testfiles[i].AssertIsEqual(downloads[i].Result); } - // Task.WaitAll(); - } - - void download(ContentId contentId, IOnlineCodexNode node, TestFile testFile) - { - var downloadedFile = node.DownloadContent(contentId); - testFile.AssertIsEqual(downloadedFile); - } - void upload(IOnlineCodexNode host, TestFile testfile, List contentIds, int pos) - { - contentIds[pos] = host.UploadFile(testfile); } } [TestFixture] From aed3edaf6aa0d90db93d3834bd0cf1b0c068970e Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Wed, 26 Apr 2023 14:59:26 +0200 Subject: [PATCH 6/7] MDF split tests into different files --- Tests/BasicTests/DownloadTests.cs | 51 ++++++++++++++++++++++++++ Tests/BasicTests/ParallelTests.cs | 61 ------------------------------- 2 files changed, 51 insertions(+), 61 deletions(-) create mode 100644 Tests/BasicTests/DownloadTests.cs diff --git a/Tests/BasicTests/DownloadTests.cs b/Tests/BasicTests/DownloadTests.cs new file mode 100644 index 0000000..bba4da1 --- /dev/null +++ b/Tests/BasicTests/DownloadTests.cs @@ -0,0 +1,51 @@ +using DistTestCore; +using KubernetesWorkflow; +using NUnit.Framework; +namespace Tests.ParallelTests +{ + [TestFixture] + public class DownloadTests : DistTest + { + [Test] + public void ThreeNodeDownloads() + { + ParallelDownload(3, 5000.MB()); + } + [Test] + public void FiveNodeDownloads() + { + ParallelDownload(5, 1000.MB()); + } + [Test] + public void TenNodeDownloads() + { + ParallelDownload(10, 256.MB()); + } + + void ParallelDownload(int numberOfNodes, ByteSize filesize) + { + var group = SetupCodexNodes(numberOfNodes).BringOnline(); + var host = SetupCodexNodes(1).BringOnline()[0]; + + foreach (var node in group) + { + host.ConnectToPeer(node); + } + + var testFile = GenerateTestFile(filesize); + var contentId = host.UploadFile(testFile); + var list = new List>(); + + foreach (var node in group) + { + list.Add(Task.Run(() => { return node.DownloadContent(contentId); })); + } + + Task.WaitAll(list.ToArray()); + foreach (var task in list) + { + testFile.AssertIsEqual(task.Result); + } + } + } +} \ No newline at end of file diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/ParallelTests.cs index b1701bc..a603524 100644 --- a/Tests/BasicTests/ParallelTests.cs +++ b/Tests/BasicTests/ParallelTests.cs @@ -3,52 +3,6 @@ using KubernetesWorkflow; using NUnit.Framework; namespace Tests.ParallelTests { - [TestFixture] - public class DownloadTests : DistTest - { - [Test] - public void ThreeNodeDownloads() - { - ParallelDownload(3, 5000.MB()); - } - [Test] - public void FiveNodeDownloads() - { - ParallelDownload(5, 1000.MB()); - } - [Test] - public void TenNodeDownloads() - { - ParallelDownload(10, 256.MB()); - } - - void ParallelDownload(int numberOfNodes, ByteSize filesize) - { - var group = SetupCodexNodes(numberOfNodes).BringOnline(); - var host = SetupCodexNodes(1).BringOnline()[0]; - - foreach (var node in group) - { - host.ConnectToPeer(node); - } - - var testFile = GenerateTestFile(filesize); - var contentId = host.UploadFile(testFile); - var list = new List>(); - - foreach (var node in group) - { - list.Add(Task.Run(() => { return node.DownloadContent(contentId); })); - } - - Task.WaitAll(list.ToArray()); - foreach (var task in list) - { - testFile.AssertIsEqual(task.Result); - } - } - } - [TestFixture] public class UploadTests : DistTest { @@ -99,19 +53,4 @@ namespace Tests.ParallelTests } } } - [TestFixture] - public class MixedTests : DistTest - { - [Test] - public void OneDownloadOneUpload() - { - } - - public void ThreeDownloadTwoUpload() - { - } - public void FiveDownloadFiveUpload() - { - } - } } \ No newline at end of file From ec311f4802bdf27aed6d77fe61608c9306904c82 Mon Sep 17 00:00:00 2001 From: Corbo12 Date: Wed, 26 Apr 2023 15:00:09 +0200 Subject: [PATCH 7/7] MDF change upload tests file name --- Tests/BasicTests/{ParallelTests.cs => UploadTests.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Tests/BasicTests/{ParallelTests.cs => UploadTests.cs} (100%) diff --git a/Tests/BasicTests/ParallelTests.cs b/Tests/BasicTests/UploadTests.cs similarity index 100% rename from Tests/BasicTests/ParallelTests.cs rename to Tests/BasicTests/UploadTests.cs