codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extracts file manager utils to separate assembly

This commit is contained in:
benbierens 2023-09-11 10:43:27 +02:00
parent 07396644a4
commit 6915e90861
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
16 changed files with 146 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ContinuousTests/ContinuousTest.cs
View File

@ -1,6 +1,7 @@
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Logs;
using FileUtils;
using KubernetesWorkflow;
using Logging;

3
ContinuousTests/SingleTestRun.cs
View File

@ -6,6 +6,7 @@ using KubernetesWorkflow;
using NUnit.Framework.Internal;
using System.Reflection;
using static Program;
using FileUtils;
namespace ContinuousTests
{
@ -38,7 +39,7 @@ namespace ContinuousTests
nodes = CreateRandomNodes();
dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration().GetFileManagerFolder());
}
public void Run(EventWaitHandle runFinishedHandle)

1
ContinuousTests/Tests/HoldMyBeerTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore;
using FileUtils;
using NUnit.Framework;
using Utils;

1
ContinuousTests/Tests/TwoClientTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore;
using FileUtils;
using NUnit.Framework;
using Utils;

5
DistTestCore/DistTest.cs
View File

@ -3,6 +3,7 @@ using DistTestCore.Helpers;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
@ -100,9 +101,7 @@ namespace DistTestCore
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.PushFileSet();
action();
Get().FileManager.PopFileSet();
Get().FileManager.ScopedFiles(action);
}
public IOnlineCodexNode SetupCodexBootstrapNode()

1
DistTestCore/DistTestCore.csproj
View File

@ -27,6 +27,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FileUtils\FileUtils.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\Nethereum\NethereumWorkflow.csproj" />

12
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using FileUtils;
using Logging;
using Utils;
using static DistTestCore.Helpers.FullConnectivityHelper;
@ -43,7 +44,11 @@ namespace DistTestCore.Helpers
public PeerConnectionState Check(Entry from, Entry to)
{
fileManager.PushFileSet();
return fileManager.ScopedFiles(() => CheckConnectivity(from, to));
}
private PeerConnectionState CheckConnectivity(Entry from, Entry to)
{
var expectedFile = GenerateTestFile(from.Node, to.Node);
using var uploadStream = File.OpenRead(expectedFile.Filename);
@ -61,11 +66,6 @@ namespace DistTestCore.Helpers
// We consider that as no-connection for the purpose of this test.
return PeerConnectionState.NoConnection;
}
finally
{
fileManager.PopFileSet();
}
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
}

1
DistTestCore/OnlineCodexNode.cs
View File

@ -2,6 +2,7 @@
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using FileUtils;
using Logging;
using NUnit.Framework;
using Utils;

3
DistTestCore/TestLifecycle.cs
View File

@ -2,6 +2,7 @@
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using Utils;
@ -20,7 +21,7 @@ namespace DistTestCore
WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
FileManager = new FileManager(Log, configuration);
FileManager = new FileManager(Log, configuration.GetFileManagerFolder());
CodexStarter = new CodexStarter(this);
PrometheusStarter = new PrometheusStarter(this);
GrafanaStarter = new GrafanaStarter(this);

106
DistTestCore/FileManager.cs → FileUtils/FileManager.cs
View File

@ -2,15 +2,15 @@
using NUnit.Framework;
using Utils;
namespace DistTestCore
namespace FileUtils
{
public interface IFileManager
{
TestFile CreateEmptyTestFile(string label = "");
TestFile GenerateTestFile(ByteSize size, string label = "");
void DeleteAllTestFiles();
void PushFileSet();
void PopFileSet();
void ScopedFiles(Action action);
T ScopedFiles<T>(Func<T> action);
}
public class FileManager : IFileManager
@ -22,9 +22,9 @@ namespace DistTestCore
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
public FileManager(BaseLog log, Configuration configuration)
public FileManager(BaseLog log, string rootFolder)
{
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
EnsureDirectory();
this.log = log;
@ -52,12 +52,27 @@ namespace DistTestCore
DeleteDirectory();
}
public void PushFileSet()
public void ScopedFiles(Action action)
{
PushFileSet();
action();
PopFileSet();
}
public T ScopedFiles<T>(Func<T> action)
{
PushFileSet();
var result = action();
PopFileSet();
return result;
}
private void PushFileSet()
{
fileSetStack.Add(new List<TestFile>());
}
public void PopFileSet()
private void PopFileSet()
{
if (!fileSetStack.Any()) return;
var pop = fileSetStack.Last();
@ -138,81 +153,4 @@ namespace DistTestCore
Directory.Delete(folder, true);
}
}
public class TestFile
{
private readonly BaseLog log;
public TestFile(BaseLog log, string filename, string label)
{
this.log = log;
Filename = filename;
Label = label;
}
public string Filename { get; }
public string Label { get; }
public void AssertIsEqual(TestFile? actual)
{
var sw = Stopwatch.Begin(log);
try
{
AssertEqual(actual);
}
finally
{
sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
}
}
public string Describe()
{
var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
return $"'{Filename}'{sizePostfix}";
}
private void AssertEqual(TestFile? actual)
{
if (actual == null) Assert.Fail("TestFile is null.");
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
var bytesExpected = new byte[FileManager.ChunkSize];
var bytesActual = new byte[FileManager.ChunkSize];
var readExpected = 0;
var readActual = 0;
while (true)
{
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
if (readExpected == 0 && readActual == 0)
{
log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
return;
}
Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
for (var i = 0; i < readActual; i++)
{
if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
}
}
}
private long GetFileSize()
{
var info = new FileInfo(Filename);
return info.Length;
}
}
}

14
FileUtils/FileUtils.csproj Normal file
View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\Utils\Utils.csproj" />
</ItemGroup>
</Project>

83
FileUtils/TestFile.cs Normal file
View File

@ -0,0 +1,83 @@
using Logging;
using NUnit.Framework;
using Utils;
namespace FileUtils
{
public class TestFile
{
private readonly BaseLog log;
public TestFile(BaseLog log, string filename, string label)
{
this.log = log;
Filename = filename;
Label = label;
}
public string Filename { get; }
public string Label { get; }
public void AssertIsEqual(TestFile? actual)
{
var sw = Stopwatch.Begin(log);
try
{
AssertEqual(actual);
}
finally
{
sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
}
}
public string Describe()
{
var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
return $"'{Filename}'{sizePostfix}";
}
private void AssertEqual(TestFile? actual)
{
if (actual == null) Assert.Fail("TestFile is null.");
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
var bytesExpected = new byte[FileManager.ChunkSize];
var bytesActual = new byte[FileManager.ChunkSize];
var readExpected = 0;
var readActual = 0;
while (true)
{
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
if (readExpected == 0 && readActual == 0)
{
log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
return;
}
Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
for (var i = 0; i < readActual; i++)
{
if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
}
}
}
private long GetFileSize()
{
var info = new FileInfo(Filename);
return info.Length;
}
}
}

1
LongTests/BasicTests/DownloadTests.cs
View File

@ -1,4 +1,5 @@
using DistTestCore;
using FileUtils;
using NUnit.Framework;
using Utils;

1
LongTests/BasicTests/UploadTests.cs
View File

@ -1,4 +1,5 @@
using DistTestCore;
using FileUtils;
using NUnit.Framework;
using Utils;

2
Tests/BasicTests/ExampleTests.cs
View File

@ -10,7 +10,7 @@ namespace Tests.BasicTests
[Test]
public void CodexLogExample()
{
var primary = SetupCodexNode();
var primary = SetupCodexNodes(2)[0];
primary.UploadFile(GenerateTestFile(5.MB()));

8
cs-codex-dist-testing.sln
View File

@ -23,7 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNe
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FileUtils", "FileUtils\FileUtils.csproj", "{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -75,6 +77,10 @@ Global
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU
{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE