status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-23 13:38:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better file manager

Browse Source
This commit is contained in:
Ben 2024-07-01 15:59:08 +02:00
parent 8d32758918
commit 926e323569
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
6 changed files with 201 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
Framework/FileUtils/FileManager.cs
View File

@ -7,27 +7,35 @@ namespace FileUtils
{
TrackedFile CreateEmptyFile(string label = "");
TrackedFile GenerateFile(ByteSize size, string label = "");
TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
void DeleteAllFiles();
void ScopedFiles(Action action);
T ScopedFiles<T>(Func<T> action);
}
public interface IGenerateOption
{
IGenerateOption Random(ByteSize size);
IGenerateOption StringRepeat(string str, ByteSize size);
IGenerateOption StringRepeat(string str, int times);
IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
IGenerateOption ByteRepeat(byte[] bytes, int times);
}
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024 * 100;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private static readonly NumberSource folderNumberSource = new NumberSource(0);
private readonly ILog log;
private readonly string rootFolder;
private readonly string folder;
private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
public const int ChunkSize = 1024 * 1024 * 100;
public FileManager(ILog log, string rootFolder)
{
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
this.log = log;
this.rootFolder = rootFolder;
}
public TrackedFile CreateEmptyFile(string label = "")
@ -41,10 +49,15 @@ namespace FileUtils
return result;
}
public TrackedFile GenerateFile(ByteSize size, string label)
public TrackedFile GenerateFile(ByteSize size, string label = "")
{
return GenerateFile(o => o.Random(size), label);
}
public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
{
var sw = Stopwatch.Begin(log);
var result = GenerateRandomFile(size, label);
var result = RunGenerators(options, label);
sw.End($"Generated file {result.Describe()}.");
return result;
}
@ -89,26 +102,35 @@ namespace FileUtils
if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
}
private TrackedFile GenerateRandomFile(ByteSize size, string label)
private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
{
var result = CreateEmptyFile(label);
CheckSpaceAvailable(result, size);
var generators = GetGenerators(options);
CheckSpaceAvailable(result, generators.GetRequiredSpace());
GenerateFileBytes(result, size);
using var stream = new FileStream(result.Filename, FileMode.Append);
generators.Run(stream);
return result;
}
private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
{
var result = new GeneratorCollection();
options(result);
return result;
}
private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
{
var file = new FileInfo(testFile.Filename);
var drive = new DriveInfo(file.Directory!.Root.FullName);
var spaceAvailable = drive.TotalFreeSpace;
if (spaceAvailable < size.SizeInBytes)
if (spaceAvailable < requiredSize)
{
var msg = $"Not enough disk space. " +
$"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
$"{Formatter.FormatByteSize(requiredSize)} required. " +
$"{Formatter.FormatByteSize(spaceAvailable)} available.";
log.Log(msg);
@ -116,34 +138,6 @@ namespace FileUtils
}
}
private void GenerateFileBytes(TrackedFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;
int chunkSize = ChunkSize;
while (bytesLeft > 0)
{
try
{
var length = Math.Min(bytesLeft, chunkSize);
AppendRandomBytesToFile(result, length);
bytesLeft -= length;
}
catch
{
chunkSize = chunkSize / 2;
if (chunkSize < 1024) throw;
}
}
}
private void AppendRandomBytesToFile(TrackedFile result, long length)
{
var bytes = new byte[length];
random.NextBytes(bytes);
using var stream = new FileStream(result.Filename, FileMode.Append);
stream.Write(bytes, 0, bytes.Length);
}
private void EnsureDirectory()
{
Directory.CreateDirectory(folder);

141
Framework/FileUtils/Generators.cs Normal file
View File

@ -0,0 +1,141 @@
using System.Text;
using Utils;
namespace FileUtils
{
public class GeneratorCollection : IGenerateOption
{
private readonly List<IGenerator> generators = new List<IGenerator>();
public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
{
var times = size.SizeInBytes / bytes.Length;
generators.Add(new ByteRepeater(bytes, times));
return this;
}
public IGenerateOption ByteRepeat(byte[] bytes, int times)
{
generators.Add(new ByteRepeater(bytes, times));
return this;
}
public IGenerateOption Random(ByteSize size)
{
generators.Add(new RandomGenerator(size));
return this;
}
public IGenerateOption StringRepeat(string str, ByteSize size)
{
var times = size.SizeInBytes / str.Length;
generators.Add(new StringRepeater(str, times));
return this;
}
public IGenerateOption StringRepeat(string str, int times)
{
generators.Add(new StringRepeater(str, times));
return this;
}
public void Run(FileStream file)
{
foreach (var generator in generators)
{
generator.Generate(file);
}
}
public long GetRequiredSpace()
{
return generators.Sum(g => g.GetRequiredSpace());
}
}
public interface IGenerator
{
void Generate(FileStream file);
long GetRequiredSpace();
}
public class ByteRepeater : IGenerator
{
private readonly byte[] bytes;
private readonly long times;
public ByteRepeater(byte[] bytes, long times)
{
this.bytes = bytes;
this.times = times;
}
public void Generate(FileStream file)
{
for (var i = 0; i < times; i++)
{
file.Write(bytes, 0, bytes.Length);
}
}
public long GetRequiredSpace()
{
return bytes.Length * times;
}
}
public class StringRepeater : IGenerator
{
private readonly string str;
private readonly long times;
public StringRepeater(string str, long times)
{
this.str = str;
this.times = times;
}
public void Generate(FileStream file)
{
using var writer = new StreamWriter(file);
for (var i = 0; i < times; i++)
{
writer.Write(str);
}
}
public long GetRequiredSpace()
{
return Encoding.ASCII.GetBytes(str).Length * times;
}
}
public class RandomGenerator : IGenerator
{
private readonly Random random = new Random();
private readonly ByteSize size;
public RandomGenerator(ByteSize size)
{
this.size = size;
}
public void Generate(FileStream file)
{
var bytesLeft = size.SizeInBytes;
while (bytesLeft > 0)
{
var size = Math.Min(bytesLeft, FileManager.ChunkSize);
var bytes = new byte[size];
random.NextBytes(bytes);
file.Write(bytes, 0, bytes.Length);
bytesLeft -= size;
}
}
public long GetRequiredSpace()
{
return size.SizeInBytes;
}
}
}

1
Tests/CodexContinuousTests/Tests/TwoClientTest.cs
View File

@ -1,7 +1,6 @@
using CodexPlugin;
using FileUtils;
using Logging;
using Newtonsoft.Json;
using NUnit.Framework;
using Utils;

17
Tests/CodexTests/BasicTests/MarketplaceTests.cs
View File

@ -1,6 +1,7 @@
using CodexContractsPlugin;
using CodexContractsPlugin.Marketplace;
using CodexPlugin;
using FileUtils;
using GethPlugin;
using NUnit.Framework;
using Utils;
@ -46,7 +47,7 @@ namespace CodexTests.BasicTests
host.Marketplace.MakeStorageAvailable(availability);
}
var testFile = GenerateTestFile(fileSize);
var testFile = CreateFile(fileSize);
var client = StartCodex(s => s
.WithName("Client")
@ -90,7 +91,7 @@ namespace CodexTests.BasicTests
var fileSize = 10.MB();
var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
var contracts = Ci.StartCodexContracts(geth);
var testFile = GenerateTestFile(fileSize);
var testFile = CreateFile(fileSize);
var client = StartCodex(s => s
.WithName("Client")
@ -124,6 +125,18 @@ namespace CodexTests.BasicTests
testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
}
private TrackedFile CreateFile(ByteSize fileSize)
{
var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);
return GenerateTestFile(o => o
.Random(segmentSize)
.ByteRepeat(new byte[] { 0xaa }, segmentSize)
.Random(segmentSize)
.ByteRepeat(new byte[] { 0xee }, segmentSize)
);
}
private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)
{
Time.Retry(() =>

5
Tests/DistTestCore/DistTest.cs
View File

@ -117,6 +117,11 @@ namespace DistTestCore
return Get().GenerateTestFile(size, label);
}
public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
{
return Get().GenerateTestFile(options, label);
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.

5
Tests/DistTestCore/TestLifecycle.cs
View File

@ -52,6 +52,11 @@ namespace DistTestCore
return entryPoint.Tools.GetFileManager().GenerateFile(size, label);
}
public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
{
return entryPoint.Tools.GetFileManager().GenerateFile(options, label);
}
public IFileManager GetFileManager()
{
return entryPoint.Tools.GetFileManager();