Moves self-updater call to starter class.

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -6,6 +6,11 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Nethereum.Generators" Version="4.21.4" />
+    <PackageReference Include="Nethereum.Generators.Net" Version="4.21.4" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexContractsPlugin.Marketplace;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
@@ -64,7 +65,8 @@ namespace CodexContractsPlugin
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
-            var abi = extractor.ExtractMarketplaceAbi();
+            var (abi, bytecode) = extractor.ExtractMarketplaceAbiAndByteCode();
+            EnsureCompatbility(abi, bytecode);
 
             var interaction = new ContractInteractions(tools.GetLog(), gethNode);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
@@ -78,6 +80,18 @@ namespace CodexContractsPlugin
             return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
         }
 
+        private void EnsureCompatbility(string abi, string bytecode)
+        {
+            var expectedByteCode = MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant();
+
+            if (bytecode != expectedByteCode)
+            {
+                Log("Deployed contract is incompatible with current build of CodexContracts plugin. Running self-updater...");
+                var selfUpdater = new SelfUpdater();
+                selfUpdater.Update(abi, bytecode);
+            }
+        }
+
         private void Log(string msg)
         {
             tools.GetLog().Log(msg);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -31,14 +31,14 @@ namespace CodexContractsPlugin
             return marketplaceAddress;
         }
 
-        public string ExtractMarketplaceAbi()
+        public (string, string) ExtractMarketplaceAbiAndByteCode()
         {
             log.Debug();
-            var marketplaceAbi = Retry(FetchMarketplaceAbi);
-            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+            var (abi, bytecode) = Retry(FetchMarketplaceAbiAndByteCode);
+            if (string.IsNullOrEmpty(abi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
-            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
-            return marketplaceAbi;
+            log.Debug("Got Marketplace ABI: " + abi);
+            return (abi, bytecode);
         }
 
         private string FetchMarketplaceAddress()
@@ -48,7 +48,7 @@ namespace CodexContractsPlugin
             return marketplace!.address;
         }
 
-        private string FetchMarketplaceAbi()
+        private (string, string) FetchMarketplaceAbiAndByteCode()
         {
             var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
 
@@ -56,19 +56,12 @@ namespace CodexContractsPlugin
             var abi = artifact["abi"];
             var byteCode = artifact["bytecode"];
             var abiResult = abi!.ToString(Formatting.None);
-            var byteCodeResult = byteCode!.ToString(Formatting.None);
-
-            if (byteCodeResult
-                .ToLowerInvariant()
-                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
-            {
-                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
-            }
-
-            return abiResult;
+            var byteCodeResult = byteCode!.ToString(Formatting.None).ToLowerInvariant().Replace("\"", "");
+          
+            return (abiResult, byteCodeResult);
         }
 
-        private static string Retry(Func<string> fetch)
+        private static T Retry<T>(Func<T> fetch)
         {
             return Time.Retry(fetch, nameof(ContractsContainerInfoExtractor));
         }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -0,0 +1,108 @@
+namespace CodexContractsPlugin
+{
+    public class SelfUpdater
+    {
+        public void Update(string abi, string bytecode)
+        {
+            var filePath = GetMarketplaceFilePath();
+            var content = GenerateContent(abi, bytecode);
+            var contentLines = content.Split("\r\n");
+
+            var beginWith = new string[]
+            {
+                "using Nethereum.ABI.FunctionEncoding.Attributes;",
+                "using Nethereum.Contracts;",
+                "using System.Numerics;",
+                "",
+                "// Generated code, do not modify.",
+                "",
+                "#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.",
+                "namespace CodexContractsPlugin.Marketplace",
+                "{"
+            };
+
+            var endWith = new string[]
+            {
+                "}",
+                "#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable."
+            };
+
+            File.Delete(filePath);
+            File.WriteAllLines(filePath,
+                beginWith.Concat(
+                contentLines.Concat(
+                endWith))
+            );
+
+            throw new Exception("Oh no! CodexContracts were updated. Current build of CodexContractsPlugin is incompatible. " +
+                "But fear not! SelfUpdater.cs has automatically updated the plugin. Just rebuild and rerun and it should work. " +
+                "Just in case, manual update instructions are found here: 'CodexContractsPlugin/Marketplace/README.md'.");
+        }
+
+        private string GetMarketplaceFilePath()
+        {
+            var here = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var path = GetMarketplaceFile(here);
+                if (path != null) return path;
+
+                var parent = Directory.GetParent(here);
+                var up = parent?.FullName;
+                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
+                here = up;
+            }
+        }
+
+        private string? GetMarketplaceFile(string root)
+        {
+            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (File.Exists(path)) return path;
+            return null;
+        }
+
+        private string GenerateContent(string abi, string bytecode)
+        {
+            var deserializer = new Nethereum.Generators.Net.GeneratorModelABIDeserialiser();
+            var abiModel = deserializer.DeserialiseABI(abi);
+            var abiCtor = abiModel.Constructor;
+            var c = new Nethereum.Generators.CQS.ContractDeploymentCQSMessageGenerator(abiCtor, "namespace", bytecode, "Marketplace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+            var lines = "";
+            lines += c.GenerateClass();
+            lines += "\r\n";
+
+            foreach (var eventAbi in abiModel.Events)
+            {
+                var d = new Nethereum.Generators.DTOs.EventDTOGenerator(eventAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += d.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var errorAbi in abiModel.Errors)
+            {
+                var e = new Nethereum.Generators.DTOs.ErrorDTOGenerator(errorAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += e.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var funcAbi in abiModel.Functions)
+            {
+                var f = new Nethereum.Generators.DTOs.FunctionOutputDTOGenerator(funcAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                var ff = new Nethereum.Generators.CQS.FunctionCQSMessageGenerator(funcAbi, "namespace", "funcoutput", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += f.GenerateClass();
+                lines += "\r\n";
+                lines += ff.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var structAbi in abiModel.Structs)
+            {
+                var g = new Nethereum.Generators.DTOs.StructTypeGenerator(structAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += g.GenerateClass();
+                lines += "\r\n";
+            }
+
+            return lines;
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -0,0 +1,193 @@
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using static FrameworkTests.Utils.RunLengthEncodingTests;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingRunTests
+    {
+        [Test]
+        [Combinatorial]
+        public void RunIncludes(
+            [Values(0, 1, 2, 3)] int start,
+            [Values(1, 2, 3, 4)] int length)
+        {
+            var run = new Run(start, length);
+
+            var shouldInclude = Enumerable.Range(start, length).ToArray();
+            var shouldExclude = new int[]
+            {
+                shouldInclude.Min() - 1,
+                shouldInclude.Max() + 1
+            };
+
+            foreach (var incl in shouldInclude)
+            {
+                Assert.That(run.Includes(incl));
+            }
+            foreach (var excl in shouldExclude)
+            {
+                Assert.That(!run.Includes(excl));
+            }
+        }
+
+        [Test]
+        public void RunExpandToInclude()
+        {
+            var run = new Run(2, 3);
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(run.ExpandToInclude(1), Is.False);
+            Assert.That(run.ExpandToInclude(2), Is.False);
+            Assert.That(run.ExpandToInclude(4), Is.False);
+            Assert.That(run.ExpandToInclude(6), Is.False);
+
+            Assert.That(run.ExpandToInclude(5), Is.True);
+            Assert.That(run.Includes(5));
+            Assert.That(!run.Includes(6));
+        }
+
+        [Test]
+        public void RunCanUnsetLastIndex()
+        {
+            var run = new Run(0, 3);
+            Assert.That(run.Includes(2));
+            var update = run.Unset(2);
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void RunCanSplit()
+        {
+            var run = new Run(0, 6); // 0, 1, 2, 3, 4, 5
+            var update = run.Unset(2);
+
+            Assert.That(run.Start, Is.EqualTo(0));
+            Assert.That(run.Length, Is.EqualTo(2)); // 0, 1
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+
+            Assert.That(!update.NewRuns[0].Includes(2));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(3));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(3)); // 3, 4, 5
+            Assert.That(!update.NewRuns[0].Includes(6));
+        }
+
+        [Test]
+        public void RunReplacesSelfWhenUnsetFirstIndex()
+        {
+            var run = new Run(0, 5);
+            var update = run.Unset(0);
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
+        [Test]
+        public void CanIterateIndices()
+        {
+            var run = new Run(2, 4);
+            var seen = new List<int>();
+            run.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
+        }
+    }
+
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public bool ExpandToInclude(int index)
+        {
+            if (index == (Start + Length))
+            {
+                Length++;
+                return true;
+            }
+            return false;
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
+                }
+                return new RunUpdate(
+                    newRuns: new[] { new Run(Start + 1, Length - 1) },
+                    removeRuns: new[] { this }
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
+        }
+        
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -0,0 +1,320 @@
+using Logging;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using NuGet.Frameworks;
+using NUnit.Framework;
+using System.Collections.Concurrent;
+using System.Numerics;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingTests
+    {
+        private readonly Random random = new Random();
+
+        [Test]
+        public void EmptySet()
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < 1000; i++)
+            {
+                Assert.That(set.IsSet(i), Is.False);
+            }
+
+            var calls = 0;
+            set.Iterate(i => calls++);
+            Assert.That(calls, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void SetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+
+            Assert.That(set.IsSet(index), Is.True);
+        }
+
+        [Test]
+        public void UnsetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+            set.Unset(index);
+
+            Assert.That(set.IsSet(index), Is.False);
+        }
+
+        [Test]
+        public void RandomIndices()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            AssertEqual(set, indices);
+        }
+
+        [Test]
+        public void RandomRunLengthEncoding()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            var encoded = set.RunLengthEncoded();
+            var decoded = IndexSet.FromRunLengthEncoded(encoded);
+
+            AssertEqual(decoded, indices);
+        }
+
+        [Test]
+        public void RunLengthEncoding()
+        {
+            var indices = new[] { 0, 1, 2, 4, 6, 7 };
+            var set = new IndexSet(indices);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                0, 3,
+                4, 1,
+                6, 2
+            }, encoded);
+        }
+
+        [Test]
+        public void RunLengthDecoding()
+        {
+            var encoded = new[]
+            {
+                2, 4, // 2, 3, 4, 5
+                7, 1, // 7
+                9, 2  // 9, 10
+            };
+
+            var set = IndexSet.FromRunLengthEncoded(encoded);
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[]
+            {
+                2, 3, 4, 5,
+                7,
+                9, 10
+            }, seen);
+        }
+
+        [Test]
+        public void SetIndexBeforeRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void SetIndexAfterRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(15);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtStartOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtEndOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(14);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexInRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(12);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 1,
+                13, 2
+            }, encoded);
+        }
+
+        private void AssertEqual(IndexSet set, int[] indices)
+        {
+            var max = indices.Max() + 1;
+            for (var i = 0; i < max; i++)
+            {
+                Assert.That(set.IsSet(i), Is.EqualTo(indices.Contains(i)));
+            }
+
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(indices, seen);
+        }
+
+        private int[] GenerateRandomIndices()
+        {
+            var number = 1000;
+            var max = 2000;
+            var all = Enumerable.Range(0, max).ToList();
+            var result = new List<int>();
+
+            while (all.Any() && result.Count < number)
+            {
+                result.Add(all.PickOneRandom());
+            }
+
+            all.Sort();
+            return all.ToArray();
+        }
+
+        public class IndexSet
+        {
+            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+            public IndexSet()
+            {
+            }
+
+            public IndexSet(int[] indices)
+            {
+                foreach (var i in indices) Set(i);
+            }
+
+            public static IndexSet FromRunLengthEncoded(int[] rle)
+            {
+                var set = new IndexSet();
+                for (var i = 0; i < rle.Length; i += 2)
+                {
+                    var start = rle[i];
+                    var length = rle[i + 1];
+                    set.runs.Add(start, new Run(start, length));
+                }
+
+                return set;
+            }
+
+            public bool IsSet(int index)
+            {
+                if (runs.ContainsKey(index)) return true;
+
+                var run = GetRunBefore(index);
+                if (run == null) return false;
+
+                return run.Includes(index);
+            }
+
+            public void Set(int index)
+            {
+                if (runs.ContainsKey(index)) return;
+
+                var run = GetRunBefore(index);
+                if (run == null || !run.ExpandToInclude(index))
+                {
+                    CreateNewRun(index);
+                }
+            }
+
+            public void Unset(int index)
+            {
+                if (runs.ContainsKey(index))
+                {
+                    HandleUpdate(runs[index].Unset(index));
+                }
+                else
+                {
+                    var run = GetRunBefore(index);
+                    if (run == null) return;
+                    HandleUpdate(run.Unset(index));
+                }
+            }
+
+            public void Iterate(Action<int> onIndex)
+            {
+                foreach (var run in runs.Values)
+                {
+                    run.Iterate(onIndex);
+                }
+            }
+
+            public int[] RunLengthEncoded()
+            {
+                return Encode().ToArray();
+            }
+
+            private IEnumerable<int> Encode()
+            {
+                foreach (var pair in runs)
+                {
+                    yield return pair.Value.Start;
+                    yield return pair.Value.Length;
+                }
+            }
+
+            private Run? GetRunBefore(int index)
+            {
+                Run? result = null;
+                foreach (var pair in runs)
+                {
+                    if (pair.Key < index) result = pair.Value;
+                    else return result;
+                }
+                return result;
+            }
+
+            private void HandleUpdate(RunUpdate runUpdate)
+            {
+                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            }
+
+            private void CreateNewRun(int index)
+            {
+                if (runs.ContainsKey(index + 1))
+                {
+                    var length = runs[index + 1].Length + 1;
+                    runs.Add(index, new Run(index, length));
+                    runs.Remove(index + 1);
+                }
+                else
+                {
+                    runs.Add(index, new Run(index, 1));
+                }
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -16,10 +16,10 @@ namespace AutoClient
         [Uniform("purchases", "np", "PURCHASES", false, "Number of concurrent purchases.")]
         public int NumConcurrentPurchases { get; set; } = 10;
 
-        [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")]
-        public int ContractDurationMinutes { get; set; } = 30;
+        [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
+        public int ContractDurationMinutes { get; set; } = 60 * 6;
 
-        [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")]
+        [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
         public int ContractExpiryMinutes { get; set; } = 15;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
@@ -34,6 +34,9 @@ namespace AutoClient
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
 
+        [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
+        public int FileSizeMb { get; set; } = 0;
+
         public string LogPath
         {
             get

Tools/AutoClient/ImageGenerator.cs

@@ -1,9 +1,26 @@
-namespace AutoClient
+using FileUtils;
+using Logging;
+using Utils;
+
+namespace AutoClient
 {
-    public class ImageGenerator
+    public interface IFileGenerator
     {
-        public async Task<string> GenerateImage()
+        Task<string> Generate();
+    }
+
+    public class ImageGenerator : IFileGenerator
+    {
+        private LogSplitter log;
+
+        public ImageGenerator(LogSplitter log)
         {
+            this.log = log;
+        }
+
+        public async Task<string> Generate()
+        {
+            log.Log("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 
@@ -14,4 +31,25 @@
             return filename;
         }
     }
+
+    public class RandomFileGenerator : IFileGenerator
+    {
+        private readonly ByteSize size;
+        private readonly FileManager fileManager;
+
+        public RandomFileGenerator(Configuration config, ILog log)
+        {
+            size = config.FileSizeMb.MB();
+            fileManager = new FileManager(log, config.DataPath);
+        }
+
+        public Task<string> Generate()
+        {
+            return Task.Run(() =>
+            {
+                var file = fileManager.GenerateFile(size);
+                return file.Filename;
+            });
+        }
+    }
 }

Tools/AutoClient/Program.cs

@@ -3,6 +3,7 @@ using AutoClient;
 using CodexOpenApi;
 using Core;
 using Logging;
+using Utils;
 
 public static class Program
 {
@@ -25,14 +26,14 @@ public static class Program
             new ConsoleLog()
         );
 
-        var address = new Utils.Address(
+        var address = new Address(
             host: config.CodexHost,
             port: config.CodexPort
         );
 
         log.Log($"Start. Address: {address}");
 
-        var imgGenerator = new ImageGenerator();
+        var generator = CreateGenerator(config, log);
 
         var client = new HttpClient();
         var codex = new CodexApi(client);
@@ -44,7 +45,7 @@ public static class Program
         for (var i = 0; i < config.NumConcurrentPurchases; i++)
         {
             purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, imgGenerator, cancellationToken)
+                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
             );
         }
 
@@ -60,6 +61,15 @@ public static class Program
         log.Log("Done.");
     }
 
+    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    {
+        if (config.FileSizeMb > 0)
+        {
+            return new RandomFileGenerator(config, log);
+        }
+        return new ImageGenerator(log);
+    }
+
     private static async Task CheckCodex(CodexApi codex, ILog log)
     {
         log.Log("Checking Codex...");

Tools/AutoClient/Purchaser.cs

@@ -13,10 +13,10 @@ namespace AutoClient
         private readonly Address address;
         private readonly CodexApi codex;
         private readonly Configuration config;
-        private readonly ImageGenerator generator;
+        private readonly IFileGenerator generator;
         private readonly CancellationToken ct;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, ImageGenerator generator, CancellationToken ct)
+        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
         {
             this.log = log;
             this.client = client;
@@ -50,7 +50,7 @@ namespace AutoClient
 
         private async Task<string> CreateFile()
         {
-            return await generator.GenerateImage();
+            return await generator.Generate();
         }
 
         private async Task<ContentId> UploadFile(string filename)

Tools/AutoClient/README.md

@@ -0,0 +1,34 @@
+# Codex auto-client
+
+This thing will generate files, upload them, and purchase storage for them in an endless loop.
+
+Can generate random images or random data of a specified size.
+
+## How to run
+
+- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
+
+## Configuration options
+Options can be configured via CLI option or environment variable.
+
+| CLI option              | Environment variable | Description                                                                                                         |
+|-------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------|
+| "--codex-host"          | "CODEXHOST"          | Codex Host address. (default 'http://localhost')                                                                    |
+| "--codex-port"          | "CODEXPORT"          | port number of Codex API. (8080 by default)                                                                         |
+| "--datapath"            | "DATAPATH"           | Root path where all data files will be saved.                                                                       |
+| "--purchases"           | "PURCHASES"          | Number of concurrent purchases.                                                                                     |
+| "--contract-duration"   | "CONTRACTDURATION"   | contract duration in minutes. (default 6 hours)                                                                     |
+| "--contract-expiry"     | "CONTRACTEXPIRY"     | contract expiry in minutes. (default 15 minutes)                                                                    |
+| "--num-hosts"           | "NUMHOSTS"           | Number of hosts for contract. (default 5)                                                                           |
+| "--num-hosts-tolerance" | "NUMTOL"             | Number of host tolerance for contract. (default 2)                                                                  |
+| "--price"               | "PRICE"              | Price of contract. (default 10)                                                                                     |
+| "--collateral"          | "COLLATERAL"         | Required collateral. (default 1)                                                                                    |
+| "--filesizemb"          | "FILESIZEMB"         | When greater than zero, size of file generated and uploaded. When zero, random images are used instead. (default 0) |
+
+## Timing
+
+Configuration: `purchases` controls the number of concurrently running storage requests.
+Configuration: `contract-duration` controls the duration in minutes of each storage request.
+Auto-client will create a new storage request every X minutes, where X is the contract duration divided by the number of purchases.
+(Timing may start to vary when contracts fail or time out.)

Tools/MarketInsights/MarketInsights.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>ae71e621-bb16-41b2-b6f3-c597d2d21157</UserSecretsId>