working runlength implementation and tests

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -0,0 +1,193 @@
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using static FrameworkTests.Utils.RunLengthEncodingTests;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingRunTests
+    {
+        [Test]
+        [Combinatorial]
+        public void RunIncludes(
+            [Values(0, 1, 2, 3)] int start,
+            [Values(1, 2, 3, 4)] int length)
+        {
+            var run = new Run(start, length);
+
+            var shouldInclude = Enumerable.Range(start, length).ToArray();
+            var shouldExclude = new int[]
+            {
+                shouldInclude.Min() - 1,
+                shouldInclude.Max() + 1
+            };
+
+            foreach (var incl in shouldInclude)
+            {
+                Assert.That(run.Includes(incl));
+            }
+            foreach (var excl in shouldExclude)
+            {
+                Assert.That(!run.Includes(excl));
+            }
+        }
+
+        [Test]
+        public void RunExpandToInclude()
+        {
+            var run = new Run(2, 3);
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(run.ExpandToInclude(1), Is.False);
+            Assert.That(run.ExpandToInclude(2), Is.False);
+            Assert.That(run.ExpandToInclude(4), Is.False);
+            Assert.That(run.ExpandToInclude(6), Is.False);
+
+            Assert.That(run.ExpandToInclude(5), Is.True);
+            Assert.That(run.Includes(5));
+            Assert.That(!run.Includes(6));
+        }
+
+        [Test]
+        public void RunCanUnsetLastIndex()
+        {
+            var run = new Run(0, 3);
+            Assert.That(run.Includes(2));
+            var update = run.Unset(2);
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void RunCanSplit()
+        {
+            var run = new Run(0, 6); // 0, 1, 2, 3, 4, 5
+            var update = run.Unset(2);
+
+            Assert.That(run.Start, Is.EqualTo(0));
+            Assert.That(run.Length, Is.EqualTo(2)); // 0, 1
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+
+            Assert.That(!update.NewRuns[0].Includes(2));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(3));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(3)); // 3, 4, 5
+            Assert.That(!update.NewRuns[0].Includes(6));
+        }
+
+        [Test]
+        public void RunReplacesSelfWhenUnsetFirstIndex()
+        {
+            var run = new Run(0, 5);
+            var update = run.Unset(0);
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
+        [Test]
+        public void CanIterateIndices()
+        {
+            var run = new Run(2, 4);
+            var seen = new List<int>();
+            run.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
+        }
+    }
+
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public bool ExpandToInclude(int index)
+        {
+            if (index == (Start + Length))
+            {
+                Length++;
+                return true;
+            }
+            return false;
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
+                }
+                return new RunUpdate(
+                    newRuns: new[] { new Run(Start + 1, Length - 1) },
+                    removeRuns: new[] { this }
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
+        }
+        
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -0,0 +1,320 @@
+using Logging;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using NuGet.Frameworks;
+using NUnit.Framework;
+using System.Collections.Concurrent;
+using System.Numerics;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingTests
+    {
+        private readonly Random random = new Random();
+
+        [Test]
+        public void EmptySet()
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < 1000; i++)
+            {
+                Assert.That(set.IsSet(i), Is.False);
+            }
+
+            var calls = 0;
+            set.Iterate(i => calls++);
+            Assert.That(calls, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void SetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+
+            Assert.That(set.IsSet(index), Is.True);
+        }
+
+        [Test]
+        public void UnsetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+            set.Unset(index);
+
+            Assert.That(set.IsSet(index), Is.False);
+        }
+
+        [Test]
+        public void RandomIndices()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            AssertEqual(set, indices);
+        }
+
+        [Test]
+        public void RandomRunLengthEncoding()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            var encoded = set.RunLengthEncoded();
+            var decoded = IndexSet.FromRunLengthEncoded(encoded);
+
+            AssertEqual(decoded, indices);
+        }
+
+        [Test]
+        public void RunLengthEncoding()
+        {
+            var indices = new[] { 0, 1, 2, 4, 6, 7 };
+            var set = new IndexSet(indices);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                0, 3,
+                4, 1,
+                6, 2
+            }, encoded);
+        }
+
+        [Test]
+        public void RunLengthDecoding()
+        {
+            var encoded = new[]
+            {
+                2, 4, // 2, 3, 4, 5
+                7, 1, // 7
+                9, 2  // 9, 10
+            };
+
+            var set = IndexSet.FromRunLengthEncoded(encoded);
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[]
+            {
+                2, 3, 4, 5,
+                7,
+                9, 10
+            }, seen);
+        }
+
+        [Test]
+        public void SetIndexBeforeRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void SetIndexAfterRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(15);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtStartOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtEndOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(14);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexInRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(12);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 1,
+                13, 2
+            }, encoded);
+        }
+
+        private void AssertEqual(IndexSet set, int[] indices)
+        {
+            var max = indices.Max() + 1;
+            for (var i = 0; i < max; i++)
+            {
+                Assert.That(set.IsSet(i), Is.EqualTo(indices.Contains(i)));
+            }
+
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(indices, seen);
+        }
+
+        private int[] GenerateRandomIndices()
+        {
+            var number = 1000;
+            var max = 2000;
+            var all = Enumerable.Range(0, max).ToList();
+            var result = new List<int>();
+
+            while (all.Any() && result.Count < number)
+            {
+                result.Add(all.PickOneRandom());
+            }
+
+            all.Sort();
+            return all.ToArray();
+        }
+
+        public class IndexSet
+        {
+            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+            public IndexSet()
+            {
+            }
+
+            public IndexSet(int[] indices)
+            {
+                foreach (var i in indices) Set(i);
+            }
+
+            public static IndexSet FromRunLengthEncoded(int[] rle)
+            {
+                var set = new IndexSet();
+                for (var i = 0; i < rle.Length; i += 2)
+                {
+                    var start = rle[i];
+                    var length = rle[i + 1];
+                    set.runs.Add(start, new Run(start, length));
+                }
+
+                return set;
+            }
+
+            public bool IsSet(int index)
+            {
+                if (runs.ContainsKey(index)) return true;
+
+                var run = GetRunBefore(index);
+                if (run == null) return false;
+
+                return run.Includes(index);
+            }
+
+            public void Set(int index)
+            {
+                if (runs.ContainsKey(index)) return;
+
+                var run = GetRunBefore(index);
+                if (run == null || !run.ExpandToInclude(index))
+                {
+                    CreateNewRun(index);
+                }
+            }
+
+            public void Unset(int index)
+            {
+                if (runs.ContainsKey(index))
+                {
+                    HandleUpdate(runs[index].Unset(index));
+                }
+                else
+                {
+                    var run = GetRunBefore(index);
+                    if (run == null) return;
+                    HandleUpdate(run.Unset(index));
+                }
+            }
+
+            public void Iterate(Action<int> onIndex)
+            {
+                foreach (var run in runs.Values)
+                {
+                    run.Iterate(onIndex);
+                }
+            }
+
+            public int[] RunLengthEncoded()
+            {
+                return Encode().ToArray();
+            }
+
+            private IEnumerable<int> Encode()
+            {
+                foreach (var pair in runs)
+                {
+                    yield return pair.Value.Start;
+                    yield return pair.Value.Length;
+                }
+            }
+
+            private Run? GetRunBefore(int index)
+            {
+                Run? result = null;
+                foreach (var pair in runs)
+                {
+                    if (pair.Key < index) result = pair.Value;
+                    else return result;
+                }
+                return result;
+            }
+
+            private void HandleUpdate(RunUpdate runUpdate)
+            {
+                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            }
+
+            private void CreateNewRun(int index)
+            {
+                if (runs.ContainsKey(index + 1))
+                {
+                    var length = runs[index + 1].Length + 1;
+                    runs.Add(index, new Run(index, length));
+                    runs.Remove(index + 1);
+                }
+                else
+                {
+                    runs.Add(index, new Run(index, 1));
+                }
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -16,10 +16,10 @@ namespace AutoClient
         [Uniform("purchases", "np", "PURCHASES", false, "Number of concurrent purchases.")]
         public int NumConcurrentPurchases { get; set; } = 10;
 
-        [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")]
-        public int ContractDurationMinutes { get; set; } = 30;
+        [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
+        public int ContractDurationMinutes { get; set; } = 60 * 6;
 
-        [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")]
+        [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
         public int ContractExpiryMinutes { get; set; } = 15;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
@@ -34,6 +34,9 @@ namespace AutoClient
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
 
+        [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
+        public int FileSizeMb { get; set; } = 0;
+
         public string LogPath
         {
             get

Tools/AutoClient/ImageGenerator.cs

@@ -1,9 +1,26 @@
-namespace AutoClient
+using FileUtils;
+using Logging;
+using Utils;
+
+namespace AutoClient
 {
-    public class ImageGenerator
+    public interface IFileGenerator
     {
-        public async Task<string> GenerateImage()
+        Task<string> Generate();
+    }
+
+    public class ImageGenerator : IFileGenerator
+    {
+        private LogSplitter log;
+
+        public ImageGenerator(LogSplitter log)
         {
+            this.log = log;
+        }
+
+        public async Task<string> Generate()
+        {
+            log.Log("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 
@@ -14,4 +31,25 @@
             return filename;
         }
     }
+
+    public class RandomFileGenerator : IFileGenerator
+    {
+        private readonly ByteSize size;
+        private readonly FileManager fileManager;
+
+        public RandomFileGenerator(Configuration config, ILog log)
+        {
+            size = config.FileSizeMb.MB();
+            fileManager = new FileManager(log, config.DataPath);
+        }
+
+        public Task<string> Generate()
+        {
+            return Task.Run(() =>
+            {
+                var file = fileManager.GenerateFile(size);
+                return file.Filename;
+            });
+        }
+    }
 }

Tools/AutoClient/Program.cs

@@ -3,6 +3,7 @@ using AutoClient;
 using CodexOpenApi;
 using Core;
 using Logging;
+using Utils;
 
 public static class Program
 {
@@ -25,14 +26,14 @@ public static class Program
             new ConsoleLog()
         );
 
-        var address = new Utils.Address(
+        var address = new Address(
             host: config.CodexHost,
             port: config.CodexPort
         );
 
         log.Log($"Start. Address: {address}");
 
-        var imgGenerator = new ImageGenerator();
+        var generator = CreateGenerator(config, log);
 
         var client = new HttpClient();
         var codex = new CodexApi(client);
@@ -44,7 +45,7 @@ public static class Program
         for (var i = 0; i < config.NumConcurrentPurchases; i++)
         {
             purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, imgGenerator, cancellationToken)
+                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
             );
         }
 
@@ -60,6 +61,15 @@ public static class Program
         log.Log("Done.");
     }
 
+    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    {
+        if (config.FileSizeMb > 0)
+        {
+            return new RandomFileGenerator(config, log);
+        }
+        return new ImageGenerator(log);
+    }
+
     private static async Task CheckCodex(CodexApi codex, ILog log)
     {
         log.Log("Checking Codex...");

Tools/AutoClient/Purchaser.cs

@@ -13,10 +13,10 @@ namespace AutoClient
         private readonly Address address;
         private readonly CodexApi codex;
         private readonly Configuration config;
-        private readonly ImageGenerator generator;
+        private readonly IFileGenerator generator;
         private readonly CancellationToken ct;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, ImageGenerator generator, CancellationToken ct)
+        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
         {
             this.log = log;
             this.client = client;
@@ -50,7 +50,7 @@ namespace AutoClient
 
         private async Task<string> CreateFile()
         {
-            return await generator.GenerateImage();
+            return await generator.Generate();
         }
 
         private async Task<ContentId> UploadFile(string filename)

Tools/AutoClient/README.md

@@ -0,0 +1,34 @@
+# Codex auto-client
+
+This thing will generate files, upload them, and purchase storage for them in an endless loop.
+
+Can generate random images or random data of a specified size.
+
+## How to run
+
+- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
+
+## Configuration options
+Options can be configured via CLI option or environment variable.
+
+| CLI option              | Environment variable | Description                                                                                                         |
+|-------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------|
+| "--codex-host"          | "CODEXHOST"          | Codex Host address. (default 'http://localhost')                                                                    |
+| "--codex-port"          | "CODEXPORT"          | port number of Codex API. (8080 by default)                                                                         |
+| "--datapath"            | "DATAPATH"           | Root path where all data files will be saved.                                                                       |
+| "--purchases"           | "PURCHASES"          | Number of concurrent purchases.                                                                                     |
+| "--contract-duration"   | "CONTRACTDURATION"   | contract duration in minutes. (default 6 hours)                                                                     |
+| "--contract-expiry"     | "CONTRACTEXPIRY"     | contract expiry in minutes. (default 15 minutes)                                                                    |
+| "--num-hosts"           | "NUMHOSTS"           | Number of hosts for contract. (default 5)                                                                           |
+| "--num-hosts-tolerance" | "NUMTOL"             | Number of host tolerance for contract. (default 2)                                                                  |
+| "--price"               | "PRICE"              | Price of contract. (default 10)                                                                                     |
+| "--collateral"          | "COLLATERAL"         | Required collateral. (default 1)                                                                                    |
+| "--filesizemb"          | "FILESIZEMB"         | When greater than zero, size of file generated and uploaded. When zero, random images are used instead. (default 0) |
+
+## Timing
+
+Configuration: `purchases` controls the number of concurrently running storage requests.
+Configuration: `contract-duration` controls the duration in minutes of each storage request.
+Auto-client will create a new storage request every X minutes, where X is the contract duration divided by the number of purchases.
+(Timing may start to vary when contracts fail or time out.)