diff --git a/Framework/FileUtils/FileManager.cs b/Framework/FileUtils/FileManager.cs index 10ecf241..d96d2972 100644 --- a/Framework/FileUtils/FileManager.cs +++ b/Framework/FileUtils/FileManager.cs @@ -7,27 +7,35 @@ namespace FileUtils { TrackedFile CreateEmptyFile(string label = ""); TrackedFile GenerateFile(ByteSize size, string label = ""); + TrackedFile GenerateFile(Action options, string label = ""); void DeleteAllFiles(); void ScopedFiles(Action action); T ScopedFiles(Func action); } + public interface IGenerateOption + { + IGenerateOption Random(ByteSize size); + IGenerateOption StringRepeat(string str, ByteSize size); + IGenerateOption StringRepeat(string str, int times); + IGenerateOption ByteRepeat(byte[] bytes, ByteSize size); + IGenerateOption ByteRepeat(byte[] bytes, int times); + } + public class FileManager : IFileManager { - public const int ChunkSize = 1024 * 1024 * 100; - private static NumberSource folderNumberSource = new NumberSource(0); - private readonly Random random = new Random(); + private static readonly NumberSource folderNumberSource = new NumberSource(0); private readonly ILog log; - private readonly string rootFolder; private readonly string folder; private readonly List> fileSetStack = new List>(); + public const int ChunkSize = 1024 * 1024 * 100; + public FileManager(ILog log, string rootFolder) { folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5")); this.log = log; - this.rootFolder = rootFolder; } public TrackedFile CreateEmptyFile(string label = "") @@ -41,10 +49,15 @@ namespace FileUtils return result; } - public TrackedFile GenerateFile(ByteSize size, string label) + public TrackedFile GenerateFile(ByteSize size, string label = "") + { + return GenerateFile(o => o.Random(size), label); + } + + public TrackedFile GenerateFile(Action options, string label = "") { var sw = Stopwatch.Begin(log); - var result = GenerateRandomFile(size, label); + var result = RunGenerators(options, label); sw.End($"Generated file {result.Describe()}."); return result; } @@ -89,26 +102,35 @@ namespace FileUtils if (!Directory.GetFiles(folder).Any()) DeleteDirectory(); } - private TrackedFile GenerateRandomFile(ByteSize size, string label) + private TrackedFile RunGenerators(Action options, string label) { var result = CreateEmptyFile(label); - CheckSpaceAvailable(result, size); + var generators = GetGenerators(options); + CheckSpaceAvailable(result, generators.GetRequiredSpace()); - GenerateFileBytes(result, size); + using var stream = new FileStream(result.Filename, FileMode.Append); + generators.Run(stream); return result; } - private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size) + private GeneratorCollection GetGenerators(Action options) + { + var result = new GeneratorCollection(); + options(result); + return result; + } + + private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize) { var file = new FileInfo(testFile.Filename); var drive = new DriveInfo(file.Directory!.Root.FullName); var spaceAvailable = drive.TotalFreeSpace; - if (spaceAvailable < size.SizeInBytes) + if (spaceAvailable < requiredSize) { var msg = $"Not enough disk space. " + - $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " + + $"{Formatter.FormatByteSize(requiredSize)} required. " + $"{Formatter.FormatByteSize(spaceAvailable)} available."; log.Log(msg); @@ -116,34 +138,6 @@ namespace FileUtils } } - private void GenerateFileBytes(TrackedFile result, ByteSize size) - { - long bytesLeft = size.SizeInBytes; - int chunkSize = ChunkSize; - while (bytesLeft > 0) - { - try - { - var length = Math.Min(bytesLeft, chunkSize); - AppendRandomBytesToFile(result, length); - bytesLeft -= length; - } - catch - { - chunkSize = chunkSize / 2; - if (chunkSize < 1024) throw; - } - } - } - - private void AppendRandomBytesToFile(TrackedFile result, long length) - { - var bytes = new byte[length]; - random.NextBytes(bytes); - using var stream = new FileStream(result.Filename, FileMode.Append); - stream.Write(bytes, 0, bytes.Length); - } - private void EnsureDirectory() { Directory.CreateDirectory(folder); diff --git a/Framework/FileUtils/Generators.cs b/Framework/FileUtils/Generators.cs new file mode 100644 index 00000000..b8406be5 --- /dev/null +++ b/Framework/FileUtils/Generators.cs @@ -0,0 +1,141 @@ +using System.Text; +using Utils; + +namespace FileUtils +{ + public class GeneratorCollection : IGenerateOption + { + private readonly List generators = new List(); + + public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size) + { + var times = size.SizeInBytes / bytes.Length; + generators.Add(new ByteRepeater(bytes, times)); + return this; + } + + public IGenerateOption ByteRepeat(byte[] bytes, int times) + { + generators.Add(new ByteRepeater(bytes, times)); + return this; + } + + public IGenerateOption Random(ByteSize size) + { + generators.Add(new RandomGenerator(size)); + return this; + } + + public IGenerateOption StringRepeat(string str, ByteSize size) + { + var times = size.SizeInBytes / str.Length; + generators.Add(new StringRepeater(str, times)); + return this; + } + + public IGenerateOption StringRepeat(string str, int times) + { + generators.Add(new StringRepeater(str, times)); + return this; + } + + public void Run(FileStream file) + { + foreach (var generator in generators) + { + generator.Generate(file); + } + } + + public long GetRequiredSpace() + { + return generators.Sum(g => g.GetRequiredSpace()); + } + } + + public interface IGenerator + { + void Generate(FileStream file); + long GetRequiredSpace(); + } + + public class ByteRepeater : IGenerator + { + private readonly byte[] bytes; + private readonly long times; + + public ByteRepeater(byte[] bytes, long times) + { + this.bytes = bytes; + this.times = times; + } + + public void Generate(FileStream file) + { + for (var i = 0; i < times; i++) + { + file.Write(bytes, 0, bytes.Length); + } + } + + public long GetRequiredSpace() + { + return bytes.Length * times; + } + } + + public class StringRepeater : IGenerator + { + private readonly string str; + private readonly long times; + + public StringRepeater(string str, long times) + { + this.str = str; + this.times = times; + } + + public void Generate(FileStream file) + { + using var writer = new StreamWriter(file); + for (var i = 0; i < times; i++) + { + writer.Write(str); + } + } + + public long GetRequiredSpace() + { + return Encoding.ASCII.GetBytes(str).Length * times; + } + } + + public class RandomGenerator : IGenerator + { + private readonly Random random = new Random(); + private readonly ByteSize size; + + public RandomGenerator(ByteSize size) + { + this.size = size; + } + + public void Generate(FileStream file) + { + var bytesLeft = size.SizeInBytes; + while (bytesLeft > 0) + { + var size = Math.Min(bytesLeft, FileManager.ChunkSize); + var bytes = new byte[size]; + random.NextBytes(bytes); + file.Write(bytes, 0, bytes.Length); + bytesLeft -= size; + } + } + + public long GetRequiredSpace() + { + return size.SizeInBytes; + } + } +} diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index 0916bdf9..190cbb61 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -1,7 +1,6 @@ using CodexPlugin; using FileUtils; using Logging; -using Newtonsoft.Json; using NUnit.Framework; using Utils; diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 4a0f404a..eec3849e 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -1,6 +1,7 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; using CodexPlugin; +using FileUtils; using GethPlugin; using NUnit.Framework; using Utils; @@ -46,7 +47,7 @@ namespace CodexTests.BasicTests host.Marketplace.MakeStorageAvailable(availability); } - var testFile = GenerateTestFile(fileSize); + var testFile = CreateFile(fileSize); var client = StartCodex(s => s .WithName("Client") @@ -90,7 +91,7 @@ namespace CodexTests.BasicTests var fileSize = 10.MB(); var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); - var testFile = GenerateTestFile(fileSize); + var testFile = CreateFile(fileSize); var client = StartCodex(s => s .WithName("Client") @@ -124,6 +125,18 @@ namespace CodexTests.BasicTests testFile.AssertIsEqual(downloader.DownloadContent(contractCid)); } + private TrackedFile CreateFile(ByteSize fileSize) + { + var segmentSize = new ByteSize(fileSize.SizeInBytes / 4); + + return GenerateTestFile(o => o + .Random(segmentSize) + .ByteRepeat(new byte[] { 0xaa }, segmentSize) + .Random(segmentSize) + .ByteRepeat(new byte[] { 0xee }, segmentSize) + ); + } + private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth) { Time.Retry(() => diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 7b1d09e1..7f670826 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -117,6 +117,11 @@ namespace DistTestCore return Get().GenerateTestFile(size, label); } + public TrackedFile GenerateTestFile(Action options, string label = "") + { + return Get().GenerateTestFile(options, label); + } + /// /// Any test files generated in 'action' will be deleted after it returns. /// This helps prevent large tests from filling up discs. diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 4191cd83..568eae44 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -52,6 +52,11 @@ namespace DistTestCore return entryPoint.Tools.GetFileManager().GenerateFile(size, label); } + public TrackedFile GenerateTestFile(Action options, string label = "") + { + return entryPoint.Tools.GetFileManager().GenerateFile(options, label); + } + public IFileManager GetFileManager() { return entryPoint.Tools.GetFileManager();