working runlength implementation and tests

# Conflicts:
#	Tests/CodexTests/BasicTests/MarketplaceTests.cs

.dockerignore

@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**

.github/workflows/docker-autoclient.yml

@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+

.github/workflows/docker-marketinsights.yml

@@ -0,0 +1,27 @@
+name: Docker - MarketInsights API
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/MarketInsights/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-marketinsights.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/MarketInsights/Dockerfile
+      docker_repo: codexstorage/codex-marketinsights
+    secrets: inherit
+

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath

Framework/Core/CoreInterface.cs

@@ -30,11 +30,7 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var msg = $"Downloading container log for '{container.Name}'";
-            entryPoint.Tools.GetLog().Log(msg);
-            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return new DownloadedLog(logHandler);
+            return workflow.DownloadContainerLog(container, tailLines);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/Http.cs

@@ -13,18 +13,21 @@ namespace Core
 
     internal class Http : IHttp
     {
-        private static readonly object httpLock = new object();
+        private static object lockLock = new object();
+        private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
         private readonly Action<HttpClient> onClientCreated;
+        private readonly string id;
 
-        internal Http(ILog log, ITimeSet timeSet)
-            : this(log, timeSet, DoNothing)
+        internal Http(string id, ILog log, ITimeSet timeSet)
+            : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
+            this.id = id;
             this.log = log;
             this.timeSet = timeSet;
             this.onClientCreated = onClientCreated;
@@ -63,12 +66,22 @@ namespace Core
 
         private T LockRetry<T>(Func<T> operation, Retry retry)
         {
+            var httpLock = GetLock();
             lock (httpLock)
             {
                 return retry.Run(operation);
             }
         }
 
+        private object GetLock()
+        {
+            lock (lockLock) // I had to.
+            {
+                if (!httpLocks.ContainsKey(id)) httpLocks.Add(id, new object());
+                return httpLocks[id];
+            }
+        }
+
         private HttpClient GetClient()
         {
             var client = new HttpClient();

Framework/Core/PluginTools.cs

@@ -27,9 +27,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp();
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp(string id);
     }
 
     public interface IFileTool
@@ -58,19 +58,19 @@ namespace Core
             log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(onClientCreated, TimeSet);
+            return CreateHttp(id, onClientCreated, TimeSet);
         }
 
-        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, onClientCreated);
+            return new Http(id, log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp()
+        public IHttp CreateHttp(string id)
         {
-            return new Http(log, TimeSet);
+            return new Http(id, log, TimeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/Core/TimeSet.cs

@@ -34,12 +34,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(3);
+            return TimeSpan.FromMinutes(2);
         }
 
         public TimeSpan HttpRetryTimeout()
         {
-            return TimeSpan.FromMinutes(10);
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryDelay()

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,12 +3,11 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
         public string[] EventsOverview { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
-            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
+            return Rewards.Any() || EventsOverview.Any();
         }
     }
 
@@ -17,15 +16,4 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
-
-    public class MarketAverage
-    {
-        public int NumberOfFinished { get; set; }
-        public int TimeRangeSeconds { get; set; }
-        public float Price { get; set; }
-        public float Size { get; set; }
-        public float Duration { get; set; }
-        public float Collateral { get; set; }
-        public float ProofProbability { get; set; }
-    }
 }

Framework/FileUtils/FileManager.cs

@@ -7,27 +7,35 @@ namespace FileUtils
     {
         TrackedFile CreateEmptyFile(string label = "");
         TrackedFile GenerateFile(ByteSize size, string label = "");
+        TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
         void DeleteAllFiles();
         void ScopedFiles(Action action);
         T ScopedFiles<T>(Func<T> action);
     }
 
+    public interface IGenerateOption
+    {
+        IGenerateOption Random(ByteSize size);
+        IGenerateOption StringRepeat(string str, ByteSize size);
+        IGenerateOption StringRepeat(string str, int times);
+        IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
+        IGenerateOption ByteRepeat(byte[] bytes, int times);
+    }
+
     public class FileManager : IFileManager
     {
-        public const int ChunkSize = 1024 * 1024 * 100;
-        private static NumberSource folderNumberSource = new NumberSource(0);
-        private readonly Random random = new Random();
+        private static readonly NumberSource folderNumberSource = new NumberSource(0);
         private readonly ILog log;
-        private readonly string rootFolder;
         private readonly string folder;
         private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
 
+        public const int ChunkSize = 1024 * 1024 * 100;
+
         public FileManager(ILog log, string rootFolder)
         {
             folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
 
             this.log = log;
-            this.rootFolder = rootFolder;
         }
 
         public TrackedFile CreateEmptyFile(string label = "")
@@ -41,10 +49,15 @@ namespace FileUtils
             return result;
         }
 
-        public TrackedFile GenerateFile(ByteSize size, string label)
+        public TrackedFile GenerateFile(ByteSize size, string label = "")
+        {
+            return GenerateFile(o => o.Random(size), label);
+        }
+
+        public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
         {
             var sw = Stopwatch.Begin(log);
-            var result = GenerateRandomFile(size, label);
+            var result = RunGenerators(options, label);
             sw.End($"Generated file {result.Describe()}.");
             return result;
         }
@@ -57,16 +70,27 @@ namespace FileUtils
         public void ScopedFiles(Action action)
         {
             PushFileSet();
-            action();
-            PopFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         public T ScopedFiles<T>(Func<T> action)
         {
             PushFileSet();
-            var result = action();
-            PopFileSet();
-            return result;
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         private void PushFileSet()
@@ -89,26 +113,35 @@ namespace FileUtils
             if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
         }
 
-        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
         {
             var result = CreateEmptyFile(label);
-            CheckSpaceAvailable(result, size);
+            var generators = GetGenerators(options);
+            CheckSpaceAvailable(result, generators.GetRequiredSpace());
 
-            GenerateFileBytes(result, size);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            generators.Run(stream);
             return result;
         }
 
-        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
+        {
+            var result = new GeneratorCollection();
+            options(result);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
         {
             var file = new FileInfo(testFile.Filename);
             var drive = new DriveInfo(file.Directory!.Root.FullName);
 
             var spaceAvailable = drive.TotalFreeSpace;
 
-            if (spaceAvailable < size.SizeInBytes)
+            if (spaceAvailable < requiredSize)
             {
                 var msg = $"Not enough disk space. " +
-                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(requiredSize)} required. " +
                     $"{Formatter.FormatByteSize(spaceAvailable)} available.";
 
                 log.Log(msg);
@@ -116,34 +149,6 @@ namespace FileUtils
             }
         }
 
-        private void GenerateFileBytes(TrackedFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            int chunkSize = ChunkSize;
-            while (bytesLeft > 0)
-            {
-                try
-                {
-                    var length = Math.Min(bytesLeft, chunkSize);
-                    AppendRandomBytesToFile(result, length);
-                    bytesLeft -= length;
-                }
-                catch
-                {
-                    chunkSize = chunkSize / 2;
-                    if (chunkSize < 1024) throw;
-                }
-            }
-        }
-
-        private void AppendRandomBytesToFile(TrackedFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
         private void EnsureDirectory()
         {
             Directory.CreateDirectory(folder);

Framework/FileUtils/Generators.cs

@@ -0,0 +1,141 @@
+using System.Text;
+using Utils;
+
+namespace FileUtils
+{
+    public class GeneratorCollection : IGenerateOption
+    {
+        private readonly List<IGenerator> generators = new List<IGenerator>();
+
+        public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
+        {
+            var times = size.SizeInBytes / bytes.Length;
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption ByteRepeat(byte[] bytes, int times)
+        {
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption Random(ByteSize size)
+        {
+            generators.Add(new RandomGenerator(size));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, ByteSize size)
+        {
+            var times = size.SizeInBytes / str.Length;
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, int times)
+        {
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public void Run(FileStream file)
+        {
+            foreach (var generator in generators)
+            {
+                generator.Generate(file);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return generators.Sum(g => g.GetRequiredSpace());
+        }
+    }
+
+    public interface IGenerator
+    {
+        void Generate(FileStream file);
+        long GetRequiredSpace();
+    }
+
+    public class ByteRepeater : IGenerator
+    {
+        private readonly byte[] bytes;
+        private readonly long times;
+
+        public ByteRepeater(byte[] bytes, long times)
+        {
+            this.bytes = bytes;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            for (var i = 0; i < times; i++)
+            {
+                file.Write(bytes, 0, bytes.Length);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return bytes.Length * times;
+        }
+    }
+
+    public class StringRepeater : IGenerator
+    {
+        private readonly string str;
+        private readonly long times;
+
+        public StringRepeater(string str, long times)
+        {
+            this.str = str;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            using var writer = new StreamWriter(file);
+            for (var i = 0; i < times; i++)
+            {
+                writer.Write(str);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return Encoding.ASCII.GetBytes(str).Length * times;
+        }
+    }
+
+    public class RandomGenerator : IGenerator
+    {
+        private readonly Random random = new Random();
+        private readonly ByteSize size;
+
+        public RandomGenerator(ByteSize size)
+        {
+            this.size = size;
+        }
+
+        public void Generate(FileStream file)
+        {
+            var bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var size = Math.Min(bytesLeft, FileManager.ChunkSize);
+                var bytes = new byte[size];
+                random.NextBytes(bytes);
+                file.Write(bytes, 0, bytes.Length);
+                bytesLeft -= size;
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return size.SizeInBytes;
+        }
+    }
+}

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -1,13 +1,15 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
-namespace Core
+namespace KubernetesWorkflow
 {
     public interface IDownloadedLog
     {
-        void IterateLines(Action<string> action);
+        string ContainerName { get; }
+
+        void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
+        string GetFilepath();
         void DeleteFile();
     }
 
@@ -15,12 +17,15 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(WriteToFileLogHandler logHandler)
+        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
         {
             logFile = logHandler.LogFile;
+            ContainerName = containerName;
         }
 
-        public void IterateLines(Action<string> action)
+        public string ContainerName { get; }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
@@ -28,7 +33,10 @@ namespace Core
             var line = streamReader.ReadLine();
             while (line != null)
             {
-                action(line);
+                if (thatContain.All(line.Contains))
+                {
+                    action(line);
+                }
                 line = streamReader.ReadLine();
             }
         }
@@ -72,6 +80,11 @@ namespace Core
             return result.ToArray();
         }
 
+        public string GetFilepath()
+        {
+            return logFile.FullFilename;
+        }
+
         public void DeleteFile()
         {
             File.Delete(logFile.FullFilename);

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace KubernetesWorkflow.Recipe
 {
@@ -21,14 +22,13 @@ namespace KubernetesWorkflow.Recipe
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
             if (userData == null) return default;
-            var jobject = (JObject)userData.UserData;
-            return jobject.ToObject<T>();
+            return JsonConvert.DeserializeObject<T>(userData.UserData);
         }
 
         private static Additional ConvertToAdditional(object userData)
         {
             var typeName = GetTypeName(userData.GetType());
-            return new Additional(typeName, userData);
+            return new Additional(typeName, JsonConvert.SerializeObject(userData));
         }
 
         private static string GetTypeName(Type type)
@@ -41,13 +41,13 @@ namespace KubernetesWorkflow.Recipe
 
     public class Additional
     {
-        public Additional(string type, object userData)
+        public Additional(string type, string userData)
         {
             Type = type;
             UserData = userData;
         }
 
         public string Type { get; }
-        public object UserData { get; }
+        public string UserData { get; }
     }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,8 +2,9 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(DateTime recipeCreatedUtc, int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
+            RecipeCreatedUtc = recipeCreatedUtc;
             Number = number;
             NameOverride = nameOverride;
             Image = image;
@@ -31,6 +32,7 @@
             if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
+        public DateTime RecipeCreatedUtc { get; }
         public string Name { get; }
         public int Number { get; }
         public string? NameOverride { get; }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
+            var recipe = new ContainerRecipe(DateTime.UtcNow, containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         CrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace(bool wait);
         void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@@ -60,7 +61,7 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
@@ -99,11 +100,19 @@ namespace KubernetesWorkflow
 
         public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
+            if (runningPod.IsStopped) return;
+            foreach (var c in runningPod.Containers)
+            {
+                c.StopLog = DownloadContainerLog(c);
+            }
+            runningPod.IsStopped = true;
+
             K8s(controller =>
             {
                 controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
+
+            cluster.Configuration.Hooks.OnContainersStopped(runningPod);
         }
 
         public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
@@ -114,6 +123,20 @@ namespace KubernetesWorkflow
             });
         }
 
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
+        {
+            var msg = $"Downloading container log for '{container.Name}'";
+            log.Log(msg);
+            var logHandler = new WriteToFileLogHandler(log, msg);
+
+            K8s(controller =>
+            {
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+            });
+
+            return new DownloadedLog(logHandler, container.Name);
+        }
+
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
         {
             return K8s(controller =>
@@ -147,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(name, r, addresses);
+                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,16 +7,19 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            Id = id;
             Name = name;
             Recipe = recipe;
             Addresses = addresses;
         }
 
+        public string Id { get; }
         public string Name { get; }
         public ContainerRecipe Recipe { get; }
         public ContainerAddress[] Addresses { get; }
+        public IDownloadedLog? StopLog { get; internal set; }
 
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
@@ -50,5 +53,21 @@ namespace KubernetesWorkflow.Types
             }
             throw new Exception("Running location not known.");
         }
+
+        public override string ToString()
+        {
+            return Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningContainer container &&
+                   Id == container.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningPod
     {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
+            Id = id;
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
@@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types
             foreach (var c in containers) c.RunningPod = this;
         }
 
+        public string Id { get; }
         public StartupConfig StartupConfig { get; }
         public StartResult StartResult { get; }
         public RunningContainer[] Containers { get; }
@@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types
             get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
         }
 
+        [JsonIgnore]
+        public bool IsStopped { get; internal set; }
+
         public string Describe()
         {
             return string.Join(",", Containers.Select(c => c.Name));
         }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningPod pod &&
+                   Id == pod.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+
+        public override string ToString()
+        {
+            if (IsStopped) return Name + " (*)";
+            return Name;
+        }
     }
 
     public static class RunningContainersExtensions

Framework/Logging/BaseLog.cs

@@ -77,7 +77,7 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
-        private string ApplyReplacements(string str)
+        protected string ApplyReplacements(string str)
         {
             if (IsDebug) return str;
             foreach (var replacement in replacements)

Framework/Logging/ConsoleLog.cs

@@ -9,7 +9,7 @@
 
         public override void Log(string message)
         {
-            Console.WriteLine(message);
+            Console.WriteLine(ApplyReplacements(message));
         }
     }
 }

Framework/OverwatchTranscript/ActionQueue.cs

@@ -0,0 +1,61 @@
+namespace OverwatchTranscript
+{
+    public class ActionQueue
+    {
+        // Using ConcurrentQueue<> here would make this process slower.
+        private readonly object queueLock = new object();
+        private readonly AutoResetEvent signal = new AutoResetEvent(false);
+        private List<Action> queue = new List<Action>();
+        private Task queueWorker = null!;
+        private bool stopping = false;
+
+        public void Start()
+        {
+            queueWorker = Task.Run(QueueWorker);
+        }
+
+        public int Count { get; private set; }
+
+        public void StopAndJoin()
+        {
+            stopping = true;
+            queueWorker.Wait();
+            if (queue.Count > 0) throw new Exception("not all acions handled");
+            queueWorker.Dispose();
+        }
+
+        public void Add(Action action)
+        {
+            if (stopping) throw new Exception("queue stopping");
+
+            lock (queueLock)
+            {
+                queue.Add(action);
+                Count = queue.Count;
+            }
+            signal.Set();
+        }
+
+        private void QueueWorker()
+        {
+            while (true)
+            {
+                signal.WaitOne(10);
+
+                List<Action> work = null!;
+                lock (queueLock)
+                {
+                    work = queue;
+                    queue = new List<Action>();
+                    Count = 0;
+                }
+                if (stopping && !work.Any()) return;
+                
+                foreach (var action in work)
+                {
+                    action();
+                }
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/BucketSet.cs

@@ -0,0 +1,110 @@
+using Logging;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public class BucketSet
+    {
+        private const int numberOfActiveBuckets = 10;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private readonly object _bucketLock = new object();
+        private readonly List<EventBucketWriter> fullBuckets = new List<EventBucketWriter>();
+        private readonly List<EventBucketWriter> activeBuckets = new List<EventBucketWriter>();
+        private readonly ActionQueue queue = new ActionQueue();
+        private int activeBucketIndex = 0;
+        private bool closed = false;
+        private string internalErrors = string.Empty;
+        
+        public BucketSet(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+
+            for (var i = 0; i < numberOfActiveBuckets;i++)
+            {
+                AddNewBucket();
+            }
+
+            queue.Start();
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            if (closed) throw new Exception("Buckets already closed!");
+            queue.Add(() => AddInternal(utc, payload));
+            
+            if (queue.Count > 1000)
+            {
+                Thread.Sleep(1);
+            }
+        }
+
+        public IFinalizedBucket[] FinalizeBuckets()
+        {
+            closed = true;
+            queue.StopAndJoin();
+
+            if (IsEmpty()) throw new Exception("No entries have been added.");
+            if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors);
+
+            var buckets = fullBuckets.Concat(activeBuckets).ToArray();
+            log.Debug($"Finalizing {buckets.Length} buckets...");
+
+            var finalized = new ConcurrentBag<IFinalizedBucket>();
+            var tasks = Parallel.ForEach(buckets, b => finalized.Add(b.FinalizeBucket()));
+            if (!tasks.IsCompleted) throw new Exception("Failed to finalize buckets: " + tasks);
+
+            return finalized.ToArray();
+        }
+
+        private bool IsEmpty()
+        {
+            return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
+        }
+
+        private void AddInternal(DateTime utc, object payload)
+        {
+            try
+            {
+                lock (_bucketLock)
+                {
+                    var current = activeBuckets[activeBucketIndex];
+                    current.Add(utc, payload);
+                    activeBucketIndex = (activeBucketIndex + 1) % numberOfActiveBuckets;
+
+                    if (current.IsFull)
+                    {
+                        log.Debug("Bucket is full. New bucket...");
+                        fullBuckets.Add(current);
+                        activeBuckets.Remove(current);
+                        AddNewBucket();
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                internalErrors += ex.ToString();
+                log.Error(ex.ToString());
+            }
+        }
+
+        private static int bucketSizeIndex = 0;
+        private static int[] bucketSizes = new[]
+        {
+            10000,
+            15000,
+            20000,
+        };
+
+        private void AddNewBucket()
+        {
+            lock (_bucketLock)
+            {
+                var size = bucketSizes[bucketSizeIndex];
+                bucketSizeIndex = (bucketSizeIndex + 1) % bucketSizes.Length;
+                activeBuckets.Add(new EventBucketWriter(log, Path.Combine(workingDir, Guid.NewGuid().ToString()), size));
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -0,0 +1,143 @@
+using Logging;
+using Newtonsoft.Json;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface IFinalizedBucket
+    {
+        bool IsEmpty { get; }
+        DateTime? SeeTopUtc();
+        BucketTop? TakeTop();
+    }
+
+    public class BucketTop
+    {
+        public BucketTop(DateTime utc, OverwatchEvent[] events)
+        {
+            Utc = utc;
+            Events = events;
+        }
+
+        public DateTime Utc { get; }
+        public OverwatchEvent[] Events { get; }
+    }
+
+    public class EventBucketReader : IFinalizedBucket
+    {
+        private readonly string bucketFile;
+        private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
+        private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
+        private bool stopping;
+
+        public EventBucketReader(ILog log, string bucketFile)
+        {
+            this.bucketFile = bucketFile;
+            if (!File.Exists(bucketFile)) throw new Exception("Doesn't exist: " + bucketFile);
+
+            log.Debug("Read Bucket open: " + bucketFile);
+
+            Task.Run(ReadBucket);
+        }
+        
+        public bool IsEmpty { get; private set; }
+
+        public DateTime? SeeTopUtc()
+        {
+            if (IsEmpty) return null;
+            while (true)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return null;
+                if (topQueue.TryPeek(out BucketTop? top))
+                {
+                    return top.Utc;
+                }
+            }
+        }
+
+        public BucketTop? TakeTop()
+        {
+            if (IsEmpty) return null;
+
+            while (true)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return null;
+                if (topQueue.TryDequeue(out BucketTop? top))
+                {
+                    itemDequeued.Set();
+                    return top;
+                }
+            }
+        }
+
+        private void ReadBucket()
+        {
+            using var file = File.OpenRead(bucketFile);
+            using var reader = new StreamReader(file);
+
+            while (true)
+            {
+                while (topQueue.Count < 5)
+                {
+                    var top = CreateNewTop(reader);
+                    if (top != null)
+                    {
+                        topQueue.Enqueue(top);
+                    }
+                    else
+                    {
+                        stopping = true;
+                        return;
+                    }
+                }
+
+                itemDequeued.Reset();
+                itemDequeued.WaitOne();
+            }
+        }
+
+        private void UpdateIsEmpty()
+        {
+            var empty = stopping && topQueue.IsEmpty;
+            if (!IsEmpty && empty)
+            {
+                File.Delete(bucketFile);
+                IsEmpty = true;
+            }
+        }
+
+        private EventBucketEntry? nextEntry = null;
+        private BucketTop? CreateNewTop(StreamReader reader)
+        {
+            if (nextEntry == null)
+            {
+                nextEntry = ReadEntry(reader);
+                if (nextEntry == null) return null;
+            }
+
+            var topEntry = nextEntry;
+            var entries = new List<EventBucketEntry>
+            {
+                topEntry
+            };
+
+            nextEntry = ReadEntry(reader);
+            while (nextEntry != null && nextEntry.Utc == topEntry.Utc)
+            {
+                entries.Add(nextEntry);
+                nextEntry = ReadEntry(reader);
+            }
+
+            return new BucketTop(topEntry.Utc, entries.Select(e => e.Event).ToArray());
+        }
+
+        private EventBucketEntry? ReadEntry(StreamReader reader)
+        {
+            var line = reader.ReadLine();
+            if (string.IsNullOrEmpty(line)) return null;
+            return JsonConvert.DeserializeObject<EventBucketEntry>(line);
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketWriter.cs

@@ -0,0 +1,113 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class EventBucketWriter
+    {
+        private const int MaxBuffer = 1000;
+
+        private readonly object _lock = new object();
+        private bool closed = false;
+        private readonly ILog log;
+        private readonly string bucketFile;
+        private readonly int maxCount;
+        private readonly List<EventBucketEntry> buffer = new List<EventBucketEntry>();
+
+        public EventBucketWriter(ILog log, string bucketFile, int maxCount)
+        {
+            this.log = log;
+            this.bucketFile = bucketFile;
+            this.maxCount = maxCount;
+            if (File.Exists(bucketFile)) throw new Exception("Already exists");
+
+            log.Debug("Write Bucket open: " + bucketFile);
+        }
+
+        public int Count { get; private set; }
+        public bool IsFull { get; private set; }
+
+        public void Add(DateTime utc, object payload)
+        {
+            lock (_lock)
+            {
+                if (closed) throw new Exception("Already closed");
+                AddToBuffer(utc, payload);
+                BufferToFile(emptyBuffer: false);
+            }
+        }
+
+        public IFinalizedBucket FinalizeBucket()
+        {
+            lock (_lock)
+            {
+                closed = true;
+                BufferToFile(emptyBuffer: true);
+                SortFileByTimestamps();
+            }
+            log.Debug($"Finalized bucket with {Count} entries");
+            return new EventBucketReader(log, bucketFile);
+        }
+
+        public override string ToString()
+        {
+            return $"EventBucket: " + Count;
+        }
+
+        private void AddToBuffer(DateTime utc, object payload)
+        {  
+            var typeName = payload.GetType().FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+            if (utc == default) throw new Exception("DateTimeUtc not set");
+
+            var entry = new EventBucketEntry
+            {
+                Utc = utc,
+                Event = new OverwatchEvent
+                {
+                    Type = typeName,
+                    Payload = Json.Serialize(payload)
+                }
+            };
+
+            Count++;
+            IsFull = Count > maxCount;
+
+            buffer.Add(entry);
+        }
+
+        private void BufferToFile(bool emptyBuffer)
+        {
+            if (emptyBuffer || buffer.Count > MaxBuffer)
+            {
+                using var file = File.Open(bucketFile, FileMode.Append);
+                using var writer = new StreamWriter(file);
+                foreach (var entry in buffer)
+                {
+                    writer.WriteLine(Json.Serialize(entry));
+                }
+                log.Debug($"Bucket wrote {buffer.Count} entries to file.");
+                buffer.Clear();
+            }
+        }
+
+        private void SortFileByTimestamps()
+        {
+            var lines = File.ReadAllLines(bucketFile);
+            var entries = lines.Select(Json.Deserialize<EventBucketEntry>)
+                .Cast<EventBucketEntry>()
+                .OrderBy(e => e.Utc)
+                .ToArray();
+
+            File.Delete(bucketFile);
+            File.WriteAllLines(bucketFile, entries.Select(e => Json.Serialize(e)));
+        }
+    }
+
+    [Serializable]
+    public class EventBucketEntry
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent Event { get; set; } = new();
+    }
+}

Framework/OverwatchTranscript/Json.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System.Globalization;
+
+namespace OverwatchTranscript
+{
+    public static class Json
+    {
+        private static JsonSerializerSettings settings = new JsonSerializerSettings
+        {
+            Formatting = Formatting.None,
+            NullValueHandling = NullValueHandling.Ignore,
+            Culture = CultureInfo.InvariantCulture,
+            DateFormatHandling = DateFormatHandling.IsoDateFormat,
+            FloatFormatHandling = FloatFormatHandling.Symbol
+        };
+
+        public static string Serialize(object obj, Formatting formatting = Formatting.None)
+        {
+            return JsonConvert.SerializeObject(obj, formatting, settings);
+        }
+
+        public static T Deserialize<T>(string json)
+        {
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}

Framework/OverwatchTranscript/Model.cs

@@ -0,0 +1,56 @@
+namespace OverwatchTranscript
+{
+    [Serializable]
+    public class OverwatchTranscript
+    {
+        public OverwatchHeader Header { get; set; } = new();
+        public OverwatchMomentReference[] MomentReferences { get; set; } = Array.Empty<OverwatchMomentReference>();
+    }
+
+    [Serializable]
+    public class OverwatchMomentReference
+    {
+        public string MomentsFile { get; set; } = string.Empty;
+        public int NumberOfMoments { get; set; }
+        public int NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeader
+    {
+        public OverwatchCommonHeader Common { get; set; } = new();
+        public OverwatchHeaderEntry[] Entries { get; set; } = Array.Empty<OverwatchHeaderEntry>();
+    }
+
+    [Serializable]
+    public class OverwatchCommonHeader
+    {
+        public long NumberOfMoments { get; set; }
+        public long NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeaderEntry
+    {
+        public string Key { get; set; } = string.Empty;
+        public string Value { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class OverwatchMoment
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent[] Events { get; set; } = Array.Empty<OverwatchEvent>();
+    }
+
+    [Serializable]
+    public class OverwatchEvent
+    {
+        public string Type { get; set; } = string.Empty;
+        public string Payload { get; set; } = string.Empty;
+    }
+}

Framework/OverwatchTranscript/MomentReader.cs

@@ -0,0 +1,104 @@
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReader
+    {
+        private readonly OverwatchTranscript model;
+        private readonly string workingDir;
+        private int referenceIndex = 0;
+        private int momentsRead = 0;
+        private OpenReference currentRef;
+
+        public MomentReader(OverwatchTranscript model, string workingDir)
+        {
+            this.model = model;
+            this.workingDir = workingDir;
+
+            currentRef = CreateOpenReference();
+        }
+
+        public OverwatchMoment? Next()
+        {
+            if (referenceIndex >= model.MomentReferences.Length) return null;
+
+            var moment = currentRef.ReadNext();
+            if (moment == null)
+            {
+                Close();
+
+                // This reference file ran out.
+                // The number of moments read should match exactly the number of moments
+                // describe in the reference. If not, error:
+                var expected = model.MomentReferences[referenceIndex].NumberOfMoments;
+                if (momentsRead != expected)
+                {
+                    throw new Exception("Number of moments read from referenced file does not match number of moments value in model. " +
+                        $"Reads: { momentsRead} - model.MomentReferences[{referenceIndex}].NumberOfMoment: {expected}"); 
+                }
+
+                referenceIndex++;
+                if (referenceIndex < model.MomentReferences.Length)
+                {
+                    // Proceed to next reference file.
+                    currentRef = CreateOpenReference();
+                    momentsRead = 0;
+                    return Next();
+                }
+                else
+                {
+                    // That was the last one.
+                    return null;
+                }
+            }
+            else
+            {
+                momentsRead++;
+                return moment;
+            }
+        }
+
+        public void Close()
+        {
+            if (currentRef != null)
+            {
+                currentRef.Close();
+                currentRef = null!;
+            }
+        }
+
+        private OpenReference CreateOpenReference()
+        {
+            var filepath = Path.Combine(workingDir, model.MomentReferences[referenceIndex].MomentsFile);
+            return new OpenReference(filepath);
+        }
+
+        private class OpenReference
+        {
+            private readonly FileStream file;
+            private readonly StreamReader reader;
+
+            public OpenReference(string filePath)
+            {
+                file = File.OpenRead(filePath);
+                reader = new StreamReader(file);
+            }
+
+            public OverwatchMoment? ReadNext()
+            {
+                var line = reader.ReadLine();
+                if (string.IsNullOrEmpty(line)) return null;
+                return JsonConvert.DeserializeObject<OverwatchMoment>(line);
+            }
+
+            public void Close()
+            {
+                reader.Close();
+                file.Close();
+
+                reader.Dispose();
+                file.Dispose();
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -0,0 +1,151 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReferenceBuilder
+    {
+        private const int MaxMomentsPerReference = 10000;
+        private readonly ILog log;
+        private readonly string workingDir;
+
+        public MomentReferenceBuilder(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+        }
+
+        public OverwatchMomentReference[] Build(IFinalizedBucket[] finalizedBuckets)
+        {
+            var result = new List<OverwatchMomentReference>();
+            var currentBuilder = new Builder(log, workingDir);
+
+            var buckets = finalizedBuckets.ToList();
+            log.Debug($"Building references for {buckets.Count} buckets.");
+            while (buckets.Any())
+            {
+                buckets.RemoveAll(b => b.IsEmpty);
+                if (!buckets.Any()) break;
+
+                var earliestUtc = GetEarliestUtc(buckets);
+                if (earliestUtc == null) continue;
+                
+                var tops = CollectAllTopsForUtc(earliestUtc.Value, buckets);
+                var moment = ConvertTopsToMoment(tops);
+                currentBuilder.Add(moment);
+                if (currentBuilder.NumberOfMoments == MaxMomentsPerReference)
+                {
+                    result.Add(currentBuilder.Build());
+                    currentBuilder = new Builder(log, workingDir);
+                }
+            }
+
+            if (currentBuilder.NumberOfMoments > 0)
+            {
+                result.Add(currentBuilder.Build());
+            }
+
+            return result.ToArray();
+        }
+
+        private OverwatchMoment ConvertTopsToMoment(List<BucketTop> tops)
+        {
+            var discintUtc = tops.Select(e => e.Utc).Distinct().ToArray();
+            if (discintUtc.Length != 1) throw new Exception("UTC mixing in moment construction.");
+
+            return new OverwatchMoment
+            {
+                Utc = tops[0].Utc,
+                Events = tops.SelectMany(e => e.Events).ToArray()
+            };
+        }
+
+        private List<BucketTop> CollectAllTopsForUtc(DateTime earliestUtc, List<IFinalizedBucket> buckets)
+        {
+            var result = new List<BucketTop>();
+
+            foreach (var bucket in buckets)
+            {
+                if (bucket.IsEmpty) continue;
+
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) continue;
+
+                if (utc.Value == earliestUtc)
+                {
+                    var top = bucket.TakeTop();
+                    if (top == null) throw new Exception("top was null after top utc was not");
+                    result.Add(top);
+                }
+            }
+
+            return result;
+        }
+
+        private DateTime? GetEarliestUtc(List<IFinalizedBucket> buckets)
+        {
+            var earliest = DateTime.MaxValue;
+            foreach (var bucket in buckets)
+            {
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) return null;
+
+                if (utc.Value < earliest) earliest = utc.Value;
+            }
+            return earliest;
+        }
+
+        public class Builder
+        {
+            private readonly ILog log;
+            private readonly string workingDir;
+            private OverwatchMomentReference reference;
+            private readonly ActionQueue queue = new ActionQueue();
+
+            public Builder(ILog log, string workingDir)
+            {
+                reference = new OverwatchMomentReference
+                {
+                    MomentsFile = Guid.NewGuid().ToString(),
+                    EarliestUtc = DateTime.MaxValue,
+                    LatestUtc = DateTime.MinValue,
+                    NumberOfEvents = 0,
+                    NumberOfMoments = 0,
+                };
+                this.log = log;
+                this.workingDir = workingDir;
+                queue.Start();
+            }
+
+            public int NumberOfMoments => reference.NumberOfMoments;
+
+            public void Add(OverwatchMoment moment)
+            {
+                if (moment.Utc < reference.EarliestUtc) reference.EarliestUtc = moment.Utc;
+                if (moment.Utc > reference.LatestUtc) reference.LatestUtc = moment.Utc;
+                reference.NumberOfMoments++;
+                reference.NumberOfEvents += moment.Events.Length;
+
+                var filePath = Path.Combine(workingDir, reference.MomentsFile);
+
+                queue.Add(() =>
+                {
+                    File.AppendAllLines(filePath, new[]
+                    {
+                        Json.Serialize(moment)
+                    });
+                });
+            }
+
+            public OverwatchMomentReference Build()
+            {
+                queue.StopAndJoin();
+
+                log.Debug($"Created reference with {reference.NumberOfMoments} moments and {reference.NumberOfEvents} events...");
+                var result = reference;
+                reference = null!;
+                return result;
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/OverwatchTranscript/Transcript.cs

@@ -0,0 +1,23 @@
+using Logging;
+
+namespace OverwatchTranscript
+{
+    public static class Transcript
+    {
+        public static ITranscriptWriter NewWriter(ILog log)
+        {
+            log = new LogPrefixer(log, "(TranscriptWriter) ");
+            return new TranscriptWriter(log, NewWorkDir());
+        }
+
+        public static ITranscriptReader NewReader(string transcriptFile)
+        {
+            return new TranscriptReader(NewWorkDir(), transcriptFile);
+        }
+
+        private static string NewWorkDir()
+        {
+            return Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        }
+    }
+}

Framework/OverwatchTranscript/TranscriptConstants.cs

@@ -0,0 +1,8 @@
+namespace OverwatchTranscript
+{
+    public static class TranscriptConstants
+    {
+        public const string TranscriptFilename = "transcript.json";
+        public const string ArtifactFolderName = "artifacts";
+    }
+}

Framework/OverwatchTranscript/TranscriptReader.cs

@@ -0,0 +1,264 @@
+using Newtonsoft.Json;
+using System.IO;
+using System;
+using System.IO.Compression;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptReader
+    {
+        OverwatchCommonHeader Header { get; }
+        T GetHeader<T>(string key);
+        void AddMomentHandler(Action<ActivateMoment> handler);
+        void AddEventHandler<T>(Action<ActivateEvent<T>> handler);
+        bool Next();
+        void Close();
+    }
+
+    public class TranscriptReader : ITranscriptReader
+    {
+        private readonly object handlersLock = new object();
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly List<Action<ActivateMoment>> momentHandlers = new List<Action<ActivateMoment>>();
+        private readonly Dictionary<string, List<Action<ActivateMoment, string>>> eventHandlers = new Dictionary<string, List<Action<ActivateMoment, string>>>();
+        private readonly string workingDir;
+        private readonly OverwatchTranscript model;
+        private bool closed;
+        private long momentCounter;
+        private readonly ConcurrentQueue<OverwatchMoment> queue = new ConcurrentQueue<OverwatchMoment>();
+        private readonly Task queueFiller;
+
+        public TranscriptReader(string workingDir, string inputFilename)
+        {
+            closed = false;
+            this.workingDir = workingDir;
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+
+            model = LoadModel(inputFilename);
+
+            queueFiller = Task.Run(() => FillQueue(model, workingDir));
+        }
+
+        public OverwatchCommonHeader Header
+        {
+            get
+            {
+                CheckClosed();
+                return model.Header.Common;
+            }
+        }
+
+        public T GetHeader<T>(string key)
+        {
+            CheckClosed();
+            var value = model.Header.Entries.First(e => e.Key == key).Value;
+            return JsonConvert.DeserializeObject<T>(value)!;
+        }
+
+        public void AddMomentHandler(Action<ActivateMoment> handler)
+        {
+            CheckClosed();
+            lock (handlersLock)
+            {
+                momentHandlers.Add(handler);
+            }
+        }
+
+        public void AddEventHandler<T>(Action<ActivateEvent<T>> handler)
+        {
+            CheckClosed();
+
+            var typeName = typeof(T).FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+
+            lock (handlersLock)
+            {
+                if (eventHandlers.ContainsKey(typeName))
+                {
+                    eventHandlers[typeName].Add(CreateEventAction(handler));
+                }
+                else
+                {
+                    eventHandlers.Add(typeName, new List<Action<ActivateMoment, string>>
+                    {
+                        CreateEventAction(handler)
+                    });
+                }
+            }
+        }
+
+        private readonly object nextLock = new object();
+        private OverwatchMoment? moment = null;
+        private OverwatchMoment? next = null;
+
+        public bool Next()
+        {
+            CheckClosed();
+
+            OverwatchMoment? m = null;
+            TimeSpan? duration = null;
+            lock (nextLock)
+            {
+                if (next == null)
+                {
+                    if (!queue.TryDequeue(out moment)) return false;
+                    queue.TryDequeue(out next);
+                }
+                else
+                {
+                    moment = next;
+                    next = null;
+                    queue.TryDequeue(out next);
+                }
+
+                m = moment;
+                duration = GetMomentDuration();
+            }
+
+            ActivateMoment(moment, duration);
+
+            return true;
+        }
+
+        public void Close()
+        {
+            CheckClosed();
+            closed = true;
+
+            queueFiller.Wait();
+
+            Directory.Delete(workingDir, true);
+        }
+
+        private Action<ActivateMoment, string> CreateEventAction<T>(Action<ActivateEvent<T>> handler)
+        {
+            return (m, s) =>
+            {
+                handler(new ActivateEvent<T>(m, JsonConvert.DeserializeObject<T>(s)!));
+            };
+        }
+
+        private void FillQueue(OverwatchTranscript model, string workingDir)
+        {
+            var reader = new MomentReader(model, workingDir);
+
+            while (true)
+            {
+                if (closed)
+                {
+                    reader.Close();
+                    return;
+                }
+
+                while (queue.Count < 10)
+                {
+                    var moment = reader.Next();
+                    if (moment == null)
+                    {
+                        reader.Close();
+                        return;
+                    }
+                    queue.Enqueue(moment);
+                }
+
+                Thread.Sleep(1);
+            }
+        }
+
+        private TimeSpan? GetMomentDuration()
+        {
+            if (moment == null) return null;
+            if (next == null) return null;
+
+            return next.Utc - moment.Utc;
+        }
+
+        private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration)
+        {
+            var m = new ActivateMoment(moment.Utc, duration, momentCounter);
+
+            lock (handlersLock)
+            {
+                ActivateMomentHandlers(m);
+
+                foreach (var @event in moment.Events)
+                {
+                    ActivateEventHandlers(m, @event);
+                }
+            }
+
+            momentCounter++;
+        }
+
+        private void ActivateMomentHandlers(ActivateMoment m)
+        {
+            foreach (var handler in momentHandlers)
+            {
+                handler(m);
+            }
+        }
+
+        private void ActivateEventHandlers(ActivateMoment m, OverwatchEvent @event)
+        {
+            if (!eventHandlers.ContainsKey(@event.Type)) return;
+            var handlers = eventHandlers[@event.Type];
+
+            foreach (var handler in handlers)
+            {
+                handler(m, @event.Payload);
+            }
+        }
+
+        private OverwatchTranscript LoadModel(string inputFilename)
+        {
+            ZipFile.ExtractToDirectory(inputFilename, workingDir);
+
+            if (!File.Exists(transcriptFile))
+            {
+                closed = true;
+                throw new Exception("Is not a transcript file. Unzipped to: " + workingDir);
+            }
+
+            return JsonConvert.DeserializeObject<OverwatchTranscript>(File.ReadAllText(transcriptFile))!;
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been closed.");
+        }
+    }
+
+    public class ActivateMoment
+    {
+        public ActivateMoment(DateTime utc, TimeSpan? duration, long index)
+        {
+            Utc = utc;
+            Duration = duration;
+            Index = index;
+        }
+
+        public DateTime Utc { get; }
+        public TimeSpan? Duration { get; }
+        public long Index { get; }
+    }
+
+    public class ActivateEvent<T>
+    {
+        public ActivateEvent(ActivateMoment moment, T payload)
+        {
+            Moment = moment;
+            Payload = payload;
+        }
+
+        public ActivateMoment Moment { get; }
+        public T Payload { get; }
+    }
+}

Framework/OverwatchTranscript/TranscriptWriter.cs

@@ -0,0 +1,130 @@
+using Logging;
+using Newtonsoft.Json;
+using System.IO.Compression;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptWriter
+    {
+        void AddHeader(string key, object value);
+        void Add(DateTime utc, object payload);
+        void IncludeArtifact(string filePath);
+        void Write(string outputFilename);
+    }
+
+    public class TranscriptWriter : ITranscriptWriter
+    {
+        private readonly object _lock = new object();
+        private readonly MomentReferenceBuilder builder;
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly Dictionary<string, string> header = new Dictionary<string, string>();
+        private readonly BucketSet bucketSet;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private bool closed;
+
+        public TranscriptWriter(ILog log, string workingDir)
+        {
+            closed = false;
+            this.log = log;
+            this.workingDir = workingDir;
+            bucketSet = new BucketSet(log, workingDir);
+            builder = new MomentReferenceBuilder(log, workingDir);
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            CheckClosed();
+            bucketSet.Add(utc, payload);
+        }
+
+        public void AddHeader(string key, object value)
+        {
+            CheckClosed();
+            lock (_lock)
+            {
+                header.Add(key, Json.Serialize(value));
+            }
+        }
+
+        public void IncludeArtifact(string filePath)
+        {
+            CheckClosed();
+            if (!File.Exists(filePath)) throw new Exception("File not found: " + filePath);
+            if (!Directory.Exists(artifactsFolder)) Directory.CreateDirectory(artifactsFolder);
+            var name = Path.GetFileName(filePath);
+            File.Copy(filePath, Path.Combine(artifactsFolder, name), overwrite: false);
+        }
+
+        public void Write(string outputFilename)
+        {
+            CheckClosed();
+            closed = true;
+
+            var momentReferences = builder.Build(bucketSet.FinalizeBuckets());
+            var model = CreateModel(momentReferences);
+
+            File.WriteAllText(transcriptFile, Json.Serialize(model, Formatting.Indented));
+
+            ZipFile.CreateFromDirectory(workingDir, outputFilename);
+            log.Debug($"Transcript written to {outputFilename}");
+            log.Debug($"Common header: {Json.Serialize(model.Header.Common, Formatting.Indented)}");
+
+            Directory.Delete(workingDir, true);
+            log.Debug($"Workdir {workingDir} deleted");
+        }
+
+        private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences)
+        {
+            lock (_lock)
+            {
+                var model = new OverwatchTranscript
+                {
+                    Header = new OverwatchHeader
+                    {
+                        Common = CreateCommonHeader(momentReferences),
+                        Entries = header.Select(h =>
+                        {
+                            return new OverwatchHeaderEntry
+                            {
+                                Key = h.Key,
+                                Value = h.Value
+                            };
+                        }).ToArray()
+                    },
+                    MomentReferences = momentReferences
+                };
+
+                header.Clear();
+                return model;
+            }
+        }
+
+        private OverwatchCommonHeader CreateCommonHeader(OverwatchMomentReference[] momentReferences)
+        {
+            var moments = momentReferences.Sum(m => m.NumberOfMoments);
+            var events = momentReferences.Sum(m => m.NumberOfEvents);
+            var earliest = momentReferences.Min(m => m.EarliestUtc);
+            var latest = momentReferences.Max(m => m.LatestUtc);
+
+            return new OverwatchCommonHeader
+            {
+                NumberOfMoments = moments,
+                NumberOfEvents = events,
+                EarliestUtc = earliest,
+                LatestUtc = latest
+            };
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been written. Cannot modify or write again.");
+        }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -13,7 +13,6 @@
 
         public long SizeInBytes { get; }
 
-
         public long ToMB()
         {
             return SizeInBytes / (1024 * 1024);

Framework/Utils/RandomUtils.cs

@@ -11,5 +11,16 @@
             remainingItems.RemoveAt(i);
             return result;
         }
+
+        public static T[] Shuffled<T>(T[] items)
+        {
+            var result = new List<T>();
+            var source = items.ToList();
+            while (source.Any())
+            {
+                result.Add(RandomUtils.PickOneRandom(source));
+            }
+            return result.ToArray();
+        }
     }
 }

Framework/Utils/RollingAverage.cs

@@ -0,0 +1,28 @@
+namespace Utils
+{
+    public static class RollingAverage
+    {
+        /// <param name="currentAverage">Value of average before new value is added.</param>
+        /// <param name="newNumberOfValues">Number of values in average after new value is added.</param>
+        /// <param name="newValue">New value to be added.</param>
+        /// <returns>New average value.</returns>
+        /// <exception cref="Exception">newNumberOfValues must be 1 or greater.</exception>
+        public static float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
+        {
+            if (newNumberOfValues < 1) throw new Exception("Should be at least 1 value.");
+
+            float n = newNumberOfValues;
+            var originalValue = currentAverage;
+            var originalValueWeight = ((n - 1.0f) / n);
+            var newValueWeight = (1.0f / n);
+            return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
+        }
+
+        public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
+        {
+            float totalWeight = weight1 + weight2;
+            if (totalWeight == 0.0f) return 0.0f;
+            return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
+        }
+    }
+}

Framework/Utils/Str.cs

@@ -0,0 +1,13 @@
+namespace Utils
+{
+    public static class Str
+    {
+        public static string Between(string input, string open, string close)
+        {
+            var openIndex = input.IndexOf(open) + open.Length;
+            var closeIndex = input.LastIndexOf(close);
+
+            return input.Substring(openIndex, closeIndex - openIndex);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -10,6 +10,7 @@ namespace CodexContractsPlugin.ChainMonitor
             Request[] requests,
             RequestFulfilledEventDTO[] fulfilled,
             RequestCancelledEventDTO[] cancelled,
+            RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
             SlotFreedEventDTO[] slotFreed
             )
@@ -18,6 +19,7 @@ namespace CodexContractsPlugin.ChainMonitor
             Requests = requests;
             Fulfilled = fulfilled;
             Cancelled = cancelled;
+            Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
         }
@@ -26,6 +28,7 @@ namespace CodexContractsPlugin.ChainMonitor
         public Request[] Requests { get; }
         public RequestFulfilledEventDTO[] Fulfilled { get; }
         public RequestCancelledEventDTO[] Cancelled { get; }
+        public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
 
@@ -37,6 +40,7 @@ namespace CodexContractsPlugin.ChainMonitor
                 all.AddRange(Requests);
                 all.AddRange(Fulfilled);
                 all.AddRange(Cancelled);
+                all.AddRange(Failed);
                 all.AddRange(SlotFilled);
                 all.AddRange(SlotFreed);
                 return all.ToArray();
@@ -60,6 +64,7 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetStorageRequests(),
                 events.GetRequestFulfilledEvents(),
                 events.GetRequestCancelledEvents(),
+                events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
                 events.GetSlotFreedEvents()
             );

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 
@@ -8,12 +9,25 @@ namespace CodexContractsPlugin.ChainMonitor
 {
     public interface IChainStateChangeHandler
     {
-        void OnNewRequest(IChainStateRequest request);
-        void OnRequestFinished(IChainStateRequest request);
-        void OnRequestFulfilled(IChainStateRequest request);
-        void OnRequestCancelled(IChainStateRequest request);
-        void OnSlotFilled(IChainStateRequest request, EthAddress host, BigInteger slotIndex);
-        void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex);
+        void OnNewRequest(RequestEvent requestEvent);
+        void OnRequestFinished(RequestEvent requestEvent);
+        void OnRequestFulfilled(RequestEvent requestEvent);
+        void OnRequestCancelled(RequestEvent requestEvent);
+        void OnRequestFailed(RequestEvent requestEvent);
+        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
+        void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+    }
+
+    public class RequestEvent
+    {
+        public RequestEvent(BlockTimeEntry block, IChainStateRequest request)
+        {
+            Block = block;
+            Request = request;
+        }
+
+        public BlockTimeEntry Block { get; }
+        public IChainStateRequest Request { get; }
     }
 
     public class ChainState
@@ -28,15 +42,12 @@ namespace CodexContractsPlugin.ChainMonitor
             this.log = new LogPrefixer(log, "(ChainState) ");
             this.contracts = contracts;
             handler = changeHandler;
-            StartUtc = startUtc;
             TotalSpan = new TimeRange(startUtc, startUtc);
         }
 
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public DateTime StartUtc { get; }
-
         public void Update()
         {
             Update(DateTime.UtcNow);
@@ -92,41 +103,49 @@ namespace CodexContractsPlugin.ChainMonitor
             var newRequest = new ChainStateRequest(log, request, RequestState.New);
             requests.Add(newRequest);
 
-            handler.OnNewRequest(newRequest);
+            handler.OnNewRequest(new RequestEvent(request.Block, newRequest));
         }
 
-        private void ApplyEvent(RequestFulfilledEventDTO request)
+        private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Started);
-            handler.OnRequestFulfilled(r);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
+            handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(RequestCancelledEventDTO request)
+        private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Cancelled);
-            handler.OnRequestCancelled(r);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
+            handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(SlotFilledEventDTO request)
+        private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
-            r.Hosts.Add(request.Host, (int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFilled (host:'{request.Host}', slotIndex:{request.SlotIndex})");
-            handler.OnSlotFilled(r, request.Host, request.SlotIndex);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
+            handler.OnRequestFailed(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(SlotFreedEventDTO request)
+        private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
-            r.Hosts.RemoveHost((int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFreed (slotIndex:{request.SlotIndex})");
-            handler.OnSlotFreed(r, request.SlotIndex);
+            r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotFreedEventDTO @event)
+        {
+            var r = FindRequest(@event.RequestId);
+            if (r == null) return;
+            r.Hosts.RemoveHost((int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
@@ -137,7 +156,7 @@ namespace CodexContractsPlugin.ChainMonitor
                     && r.FinishedUtc < eventsUtc)
                 {
                     r.UpdateState(blockNumber, RequestState.Finished);
-                    handler.OnRequestFinished(r);
+                    handler.OnRequestFinished(new RequestEvent(new BlockTimeEntry(blockNumber, eventsUtc), r));
                 }
             }
         }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -0,0 +1,55 @@
+using GethPlugin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainStateChangeHandlerMux : IChainStateChangeHandler
+    {
+        public ChainStateChangeHandlerMux(params IChainStateChangeHandler[] handlers)
+        {
+            Handlers = handlers.ToList();
+        }
+
+        public List<IChainStateChangeHandler> Handlers { get; } = new List<IChainStateChangeHandler>();
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnNewRequest(requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestCancelled(requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFailed(requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFinished(requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFulfilled(requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFilled(requestEvent, host, slotIndex);
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -5,27 +5,31 @@ namespace CodexContractsPlugin.ChainMonitor
 {
     public class DoNothingChainEventHandler : IChainStateChangeHandler
     {
-        public void OnNewRequest(IChainStateRequest request)
+        public void OnNewRequest(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestCancelled(IChainStateRequest request)
+        public void OnRequestCancelled(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestFinished(IChainStateRequest request)
+        public void OnRequestFailed(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestFulfilled(IChainStateRequest request)
+        public void OnRequestFinished(RequestEvent requestEvent)
         {
         }
 
-        public void OnSlotFilled(IChainStateRequest request, EthAddress host, BigInteger slotIndex)
+        public void OnRequestFulfilled(RequestEvent requestEvent)
         {
         }
 
-        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
     }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -13,6 +13,7 @@ namespace CodexContractsPlugin
         Request[] GetStorageRequests();
         RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
         RequestCancelledEventDTO[] GetRequestCancelledEvents();
+        RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
     }
@@ -71,6 +72,17 @@ namespace CodexContractsPlugin
             }).ToArray();
         }
 
+        public RequestFailedEventDTO[] GetRequestFailedEvents()
+        {
+            var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
         public SlotFilledEventDTO[] GetSlotFilledEvents()
         {
             var events = gethNode.GetEvents<SlotFilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -40,6 +40,12 @@ namespace CodexContractsPlugin.Marketplace
         public BlockTimeEntry Block { get; set; }
     }
 
+    public partial class RequestFailedEventDTO : IHasBlock
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
     public partial class SlotFilledEventDTO : IHasBlock
     {
         [JsonIgnore]

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -132,19 +132,6 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
-        public void LogDiskSpace(string msg)
-        {
-            try
-            {
-                var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
-                Log($"{msg} - Disk info: {diskInfo}");
-            }
-            catch (Exception e)
-            {
-                Log("Failed to get disk info: " + e);
-            }
-        }
-
         public void DeleteRepoFolder()
         {
             try
@@ -163,13 +150,13 @@ namespace CodexPlugin
 
         private T OnCodex<T>(Func<CodexApi, Task<T>> action)
         {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action));
+            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
             return result;
         }
 
         private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
         {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
             return result;
         }
 
@@ -196,7 +183,7 @@ namespace CodexPlugin
         private IEndpoint GetEndpoint()
         {
             return tools
-                .CreateHttp(CheckContainerCrashed)
+                .CreateHttp(GetHttpId(), CheckContainerCrashed)
                 .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
@@ -205,6 +192,11 @@ namespace CodexPlugin
             return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
         }
 
+        private string GetHttpId()
+        {
+            return GetAddress().ToString();
+        }
+
         private void CheckContainerCrashed(HttpClient client)
         {
             if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
@@ -227,8 +219,6 @@ namespace CodexPlugin
                 $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
                 $"Checking if node responds to debug/info...");
 
-            LogDiskSpace("After retry failure");
-
             try
             {
                 var debugInfo = GetDebugInfo();

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-305b80a-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
 using FileUtils;
 using GethPlugin;
 using KubernetesWorkflow;
@@ -12,6 +13,7 @@ namespace CodexPlugin
     public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
     {
         string GetName();
+        string GetPeerId();
         DebugInfo GetDebugInfo();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
@@ -39,19 +41,36 @@ namespace CodexPlugin
     public class CodexNode : ICodexNode
     {
         private const string UploadFailedMessage = "Unable to store block";
+        private readonly ILog log;
         private readonly IPluginTools tools;
+        private readonly ICodexNodeHooks hooks;
         private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
+        private string peerId = string.Empty;
+        private string nodeId = string.Empty;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
         {
             this.tools = tools;
             this.ethAccount = ethAccount;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
+            this.hooks = hooks;
             Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
+
+            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
+        }
+
+        public void Awake()
+        {
+            hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image, ethAccount);
+        }
+
+        public void Initialize()
+        {
+            hooks.OnNodeStarted(peerId, nodeId);
         }
 
         public RunningPod Pod { get { return CodexAccess.Container; } }
@@ -95,6 +114,11 @@ namespace CodexPlugin
             return Container.Name;
         }
 
+        public string GetPeerId()
+        {
+            return peerId;
+        }
+
         public DebugInfo GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
@@ -115,27 +139,29 @@ namespace CodexPlugin
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
         {
-            CodexAccess.LogDiskSpace("Before upload");
-
             using var fileStream = File.OpenRead(file.Filename);
+            var uniqueId = Guid.NewGuid().ToString();
+            var size = file.GetFilesize();
+
+            hooks.OnFileUploading(uniqueId, size);
 
             var logMessage = $"Uploading file {file.Describe()}...";
-            Log(logMessage);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream, onFailure);
             });
 
             var response = measurement.Value;
-            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+            transferSpeeds.AddUploadSample(size, measurement.Duration);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
-            Log($"Uploaded file. Received contentId: '{response}'.");
-            CodexAccess.LogDiskSpace("After upload");
+            Log($"Uploaded file {file.Describe()}. Received contentId: '{response}'.");
 
-            return new ContentId(response);
+            var cid = new ContentId(response);
+            hooks.OnFileUploaded(uniqueId, size, cid);
+            return cid;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
@@ -145,12 +171,17 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
-            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
-            Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+            hooks.OnFileDownloading(contentId);
+            Log($"Downloading '{contentId}'...");
+
+            var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+
+            var size = file.GetFilesize();
+            transferSpeeds.AddDownloadSample(size, measurement);
+            hooks.OnFileDownloaded(size, contentId);
+
             return file;
         }
 
@@ -188,6 +219,8 @@ namespace CodexPlugin
         public void Stop(bool waitTillStopped)
         {
             Log("Stopping...");
+            hooks.OnNodeStopping();
+
             CrashWatcher.Stop();
             Group.Stop(this, waitTillStopped);
         }
@@ -195,7 +228,8 @@ namespace CodexPlugin
         public void EnsureOnlineGetVersionResponse()
         {
             var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.Id;
+            peerId = debugInfo.Id;
+            nodeId = debugInfo.Table.LocalNode.NodeId;
             var nodeName = CodexAccess.Container.Name;
 
             if (!debugInfo.Version.IsValid())
@@ -203,9 +237,10 @@ namespace CodexPlugin
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
-            var log = tools.GetLog();
-            log.AddStringReplace(nodePeerId, nodeName);
+            log.AddStringReplace(peerId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
             log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(debugInfo.Table.LocalNode.NodeId), nodeName);
             Version = debugInfo.Version;
         }
 
@@ -223,8 +258,6 @@ namespace CodexPlugin
 
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
-            CodexAccess.LogDiskSpace("Before download");
-
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
@@ -236,8 +269,6 @@ namespace CodexPlugin
                 Log($"Failed to download file '{contentId}'.");
                 throw;
             }
-
-            CodexAccess.LogDiskSpace("After download");
         }
 
         private void EnsureMarketplace()
@@ -247,7 +278,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            tools.GetLog().Log($"{GetName()}: {msg}");
+            log.Log(msg);
         }
 
         private void DoNothing(Failure failure)

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
@@ -14,23 +15,27 @@ namespace CodexPlugin
     public class CodexNodeFactory : ICodexNodeFactory
     {
         private readonly IPluginTools tools;
+        private readonly CodexHooksFactory codexHooksFactory;
 
-        public CodexNodeFactory(IPluginTools tools)
+        public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
         {
             this.tools = tools;
+            this.codexHooksFactory = codexHooksFactory;
         }
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
             var ethAccount = GetEthAccount(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
+            var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
+
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount, hooks);
+            return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount, ICodexNodeHooks hooks)
         {
             if (ethAccount == null) return new MarketplaceUnavailable();
-            return new MarketplaceAccess(tools.GetLog(), codexAccess);
+            return new MarketplaceAccess(tools.GetLog(), codexAccess, hooks);
         }
 
         private EthAccount? GetEthAccount(CodexAccess access)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -79,13 +79,16 @@ namespace CodexPlugin
             }
 
             Version = first;
+            foreach (var node in Nodes) node.Initialize();
         }
 
         private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
         {
             var watcher = factory.CreateCrashWatcher(c.Containers.Single());
             var access = new CodexAccess(tools, c, watcher);
-            return factory.CreateOnlineCodexNode(access, this);
+            var node = factory.CreateOnlineCodexNode(access, this);
+            node.Awake();
+            return node;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,3 +1,4 @@
+using CodexPlugin.Hooks;
 using Core;
 using KubernetesWorkflow.Types;
 
@@ -57,6 +58,11 @@ namespace CodexPlugin
             }
         }
 
+        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        {
+            codexStarter.HooksFactory.Provider = hooksProvider;
+        }
+
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = new CodexSetup(numberOfNodes);

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -29,6 +29,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -169,7 +169,7 @@ namespace CodexPlugin
         public bool IsValidator { get; private set; }
         public Ether InitialEth { get; private set; } = 0.Eth();
         public TestToken InitialTestTokens { get; private set; } = 0.Tst();
-        public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup();
+        public EthAccountSetup EthAccountSetup { get; } = new EthAccountSetup();
 
         public IMarketplaceSetup AsStorageNode()
         {
@@ -201,8 +201,8 @@ namespace CodexPlugin
             var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
             result += IsStorageNode ? "(storageNode)" : "()";
             result += IsValidator ? "(validator)" : "() ";
-            result += $"Address: '{EthAccountSetup}' ";
-            result += $"{InitialEth.Eth} / {InitialTestTokens}";
+            result += $"Pinned address: '{EthAccountSetup}' ";
+            result += $"{InitialEth} / {InitialTestTokens}";
             result += "] ";
             return result;
         }
@@ -235,6 +235,7 @@ namespace CodexPlugin
 
         public override string ToString()
         {
+            if (!accounts.Any()) return "NoEthAccounts";
             return string.Join(",", accounts.Select(a => a.ToString()).ToArray());
         }
     }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -1,4 +1,6 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
+using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
@@ -19,6 +21,8 @@ namespace CodexPlugin
             apiChecker = new ApiChecker(pluginTools);
         }
 
+        public CodexHooksFactory HooksFactory { get; } = new CodexHooksFactory();
+
         public RunningPod[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
@@ -34,7 +38,8 @@ namespace CodexPlugin
             {
                 var podInfo = GetPodInfo(rc);
                 var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' PodLabel: '{c.RunningPod.StartResult.Deployment.PodLabel}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
+                Log($"Started node with image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
+                LogEthAddress(rc);
             }
             LogSeparator();
 
@@ -43,7 +48,7 @@ namespace CodexPlugin
 
         public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
-            var codexNodeFactory = new CodexNodeFactory(pluginTools);
+            var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
 
             var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);
 
@@ -141,6 +146,13 @@ namespace CodexPlugin
             Log("----------------------------------------------------------------------------");
         }
 
+        private void LogEthAddress(RunningPod rc)
+        {
+            var account = rc.Containers.First().Recipe.Additionals.Get<EthAccount>();
+            if (account == null) return;
+            Log($"{rc.Name} = {account}");
+        }
+
         private void Log(string message)
         {
             pluginTools.GetLog().Log(message);

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -8,7 +8,7 @@ namespace CodexPlugin
         public string? NameOverride { get; set; }
         public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
         public CodexLogLevel LogLevel { get; set; }
-        public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn);
+        public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Warn);
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
@@ -50,10 +50,10 @@ namespace CodexPlugin
                     "secure",
                     "chronosstream",
                     "connection",
-                    "connmanager",
+                    // Removed: "connmanager", is used for transcript peer-dropped event.
                     "websock",
                     "ws-session",
-                    "dialer",
+                    // Removed: "dialer", is used for transcript successful-dial event.
                     "muxedupgrade",
                     "upgrade",
                     "identify"

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -95,6 +95,11 @@ namespace CodexPlugin
 
         public string Id { get; }
 
+        public override string ToString()
+        {
+            return Id;
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is ContentId id && Id == id.Id;

ProjectPlugins/CodexPlugin/CodexUtils.cs

@@ -0,0 +1,24 @@
+namespace CodexPlugin
+{
+    public static class CodexUtils
+    {
+        public static string ToShortId(string id)
+        {
+            if (id.Length > 10)
+            {
+                return $"{id[..3]}*{id[^6..]}";
+            }
+            return id;
+        }
+
+        // after update of codex-dht, shortID should be consistent!
+        public static string ToNodeIdShortId(string id)
+        {
+            if (id.Length > 10)
+            {
+                return $"{id[..2]}*{id[^6..]}";
+            }
+            return id;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
 using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
@@ -38,6 +39,11 @@ namespace CodexPlugin
             return ci.StartCodexNodes(number, s => { });
         }
 
+        public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
+        {
+            Plugin(ci).SetCodexHooksProvider(hooksProvider);
+        }
+
         private static CodexPlugin Plugin(CoreInterface ci)
         {
             return ci.GetPlugin<CodexPlugin>();

ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs

@@ -0,0 +1,71 @@
+using GethPlugin;
+using Utils;
+
+namespace CodexPlugin.Hooks
+{
+    public interface ICodexHooksProvider
+    {
+        ICodexNodeHooks CreateHooks(string nodeName);
+    }
+
+    public class CodexHooksFactory
+    {
+        public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
+
+        public ICodexNodeHooks CreateHooks(string nodeName)
+        {
+            return Provider.CreateHooks(nodeName);
+        }
+    }
+
+    public class DoNothingHooksProvider : ICodexHooksProvider
+    {
+        public ICodexNodeHooks CreateHooks(string nodeName)
+        {
+            return new DoNothingCodexHooks();
+        }
+    }
+
+    public class DoNothingCodexHooks : ICodexNodeHooks
+    {
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+        }
+
+        public void OnNodeStarted(string peerId, string nodeId)
+        {
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+        }
+
+        public void OnNodeStopping()
+        {
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+        }
+    }
+}

ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs

@@ -0,0 +1,19 @@
+using GethPlugin;
+using Utils;
+
+namespace CodexPlugin.Hooks
+{
+    public interface ICodexNodeHooks
+    {
+        void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
+        void OnNodeStarted(string peerId, string nodeId);
+        void OnNodeStopping();
+        void OnFileUploading(string uid, ByteSize size);
+        void OnFileUploaded(string uid, ByteSize size, ContentId cid);
+        void OnFileDownloading(ContentId cid);
+        void OnFileDownloaded(ByteSize size, ContentId cid);
+        void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
+        void OnStorageContractUpdated(StoragePurchase purchaseStatus);
+        void OnStorageAvailabilityCreated(StorageAvailability response);
+    }
+}

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,4 +1,5 @@
-using Logging;
+using CodexPlugin.Hooks;
+using Logging;
 using Utils;
 
 namespace CodexPlugin
@@ -13,11 +14,13 @@ namespace CodexPlugin
     {
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
+        private readonly ICodexNodeHooks hooks;
 
-        public MarketplaceAccess(ILog log, CodexAccess codexAccess)
+        public MarketplaceAccess(ILog log, CodexAccess codexAccess, ICodexNodeHooks hooks)
         {
             this.log = log;
             this.codexAccess = codexAccess;
+            this.hooks = hooks;
         }
 
         public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
@@ -27,6 +30,7 @@ namespace CodexPlugin
             var response = codexAccess.RequestStorage(purchase);
 
             if (string.IsNullOrEmpty(response) ||
+                response == "Unable to encode manifest" ||
                 response == "Purchasing not available" ||
                 response == "Expiry required" ||
                 response == "Expiry needs to be in future" ||
@@ -37,8 +41,11 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase);
+            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
             contract.WaitForStorageContractSubmitted();
+
+            hooks.OnStorageContractSubmitted(contract);
+
             return contract;
         }
 
@@ -49,6 +56,7 @@ namespace CodexPlugin
             var response = codexAccess.SalesAvailability(availability);
 
             Log($"Storage successfully made available. Id: {response.Id}");
+            hooks.OnStorageAvailabilityCreated(response);
 
             return response.Id;
         }

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs

@@ -0,0 +1,128 @@
+using CodexPlugin.OverwatchSupport.LineConverters;
+using KubernetesWorkflow;
+using OverwatchTranscript;
+using Utils;
+
+namespace CodexPlugin.OverwatchSupport
+{
+    public class CodexLogConverter
+    {
+        private readonly ITranscriptWriter writer;
+        private readonly CodexTranscriptWriterConfig config;
+        private readonly IdentityMap identityMap;
+
+        public CodexLogConverter(ITranscriptWriter writer, CodexTranscriptWriterConfig config, IdentityMap identityMap)
+        {
+            this.writer = writer;
+            this.config = config;
+            this.identityMap = identityMap;
+        }
+
+        public void ProcessLog(IDownloadedLog log)
+        {
+            var name = DetermineName(log);
+            var identityIndex = identityMap.GetIndex(name);
+            var runner = new ConversionRunner(writer, config, identityMap, identityIndex);
+            runner.Run(log);
+        }
+
+        private string DetermineName(IDownloadedLog log)
+        {
+            // Expected string:
+            // Downloading container log for '<Downloader1>'
+            var nameLine = log.FindLinesThatContain("Downloading container log for").First();
+            return Str.Between(nameLine, "'", "'");
+        }
+    }
+
+    public class ConversionRunner
+    {
+        private readonly ITranscriptWriter writer;
+        private readonly IdentityMap nameIdMap;
+        private readonly int nodeIdentityIndex;
+        private readonly ILineConverter[] converters;
+
+        public ConversionRunner(ITranscriptWriter writer, CodexTranscriptWriterConfig config, IdentityMap nameIdMap, int nodeIdentityIndex)
+        {
+            this.nodeIdentityIndex = nodeIdentityIndex;
+            this.writer = writer;
+            this.nameIdMap = nameIdMap;
+
+            converters = CreateConverters(config).ToArray();
+        }
+
+        private IEnumerable<ILineConverter> CreateConverters(CodexTranscriptWriterConfig config)
+        {
+            if (config.IncludeBlockReceivedEvents)
+            {
+                yield return new BlockReceivedLineConverter();
+            }
+            yield return new BootstrapLineConverter();
+            yield return new DialSuccessfulLineConverter();
+            yield return new PeerDroppedLineConverter();
+        }
+
+        public void Run(IDownloadedLog log)
+        {
+            log.IterateLines(line =>
+            {
+                foreach (var converter in converters)
+                {
+                    ProcessLine(line, converter);
+                }
+            });
+        }
+
+        private void AddEvent(DateTime utc, Action<OverwatchCodexEvent> action)
+        {
+            var e = new OverwatchCodexEvent
+            {
+                NodeIdentity = nodeIdentityIndex,
+            };
+            action(e);
+
+            e.Write(utc, writer);
+        }
+
+        private void ProcessLine(string line, ILineConverter converter)
+        {
+            if (!line.Contains(converter.Interest)) return;
+
+            var codexLine = CodexLogLine.Parse(line);
+
+            if (codexLine == null) throw new Exception("Unable to parse required line");
+            EnsureFullIds(codexLine);
+
+            converter.Process(codexLine, (action) =>
+            {
+                AddEvent(codexLine.TimestampUtc, action);
+            });
+        }
+
+        private void EnsureFullIds(CodexLogLine codexLine)
+        {
+            // The issue is: node IDs occure both in full and short version.
+            // Downstream tools will assume that a node ID string-equals its own ID.
+            // So we replace all shortened IDs we can find with their full ones.
+
+            // Usually, the shortID appears as the entire string of an attribute:
+            // "peerId=123*567890"
+            // But sometimes, it is part of a larger string:
+            // "thing=abc:123*567890,def"
+            
+            foreach (var pair in codexLine.Attributes)
+            {
+                if (pair.Value.Contains("*"))
+                {
+                    codexLine.Attributes[pair.Key] = nameIdMap.ReplaceShortIds(pair.Value);
+                }
+            }
+        }
+    }
+
+    public interface ILineConverter
+    {
+        string Interest { get; }
+        void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent);
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs

@@ -0,0 +1,191 @@
+using CodexPlugin.Hooks;
+using GethPlugin;
+using OverwatchTranscript;
+using Utils;
+
+namespace CodexPlugin.OverwatchSupport
+{
+    public class CodexNodeTranscriptWriter : ICodexNodeHooks
+    {
+        private readonly ITranscriptWriter writer;
+        private readonly IdentityMap identityMap;
+        private readonly string name;
+        private int identityIndex = -1;
+        private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
+
+        public CodexNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name)
+        {
+            this.writer = writer;
+            this.identityMap = identityMap;
+            this.name = name;
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+            WriteCodexEvent(startUtc, e =>
+            {
+                e.NodeStarting = new NodeStartingEvent
+                {
+                    Image = image,
+                    EthAddress = ethAccount != null ? ethAccount.ToString() : ""
+                };
+            });
+        }
+
+        public void OnNodeStarted(string peerId, string nodeId)
+        {
+            if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
+            {
+                throw new Exception("Node started - peerId and/or nodeId unknown.");
+            }
+
+            identityMap.Add(name, peerId, nodeId);
+            identityIndex = identityMap.GetIndex(name);
+
+            WriteCodexEvent(e =>
+            {
+                e.NodeStarted = new NodeStartedEvent
+                {
+                };
+            });
+        }
+
+        public void OnNodeStopping()
+        {
+            WriteCodexEvent(e =>
+            {
+                e.NodeStopping = new NodeStoppingEvent
+                {
+                };
+            });
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.FileDownloading = new FileDownloadingEvent
+                {
+                    Cid = cid.Id
+                };
+            });
+        }
+
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.FileDownloaded = new FileDownloadedEvent
+                {
+                    Cid = cid.Id,
+                    ByteSize = size.SizeInBytes
+                };
+            });
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.FileUploading = new FileUploadingEvent
+                {
+                    UniqueId = uid,
+                    ByteSize = size.SizeInBytes
+                };
+            });
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.FileUploaded = new FileUploadedEvent
+                { 
+                    UniqueId = uid,
+                    Cid = cid.Id,
+                    ByteSize = size.SizeInBytes
+                };
+            });
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.StorageContractSubmitted = new StorageContractSubmittedEvent
+                {
+                    PurchaseId = storagePurchaseContract.PurchaseId,
+                    PurchaseRequest = storagePurchaseContract.Purchase
+                };
+            });
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.StorageContractUpdated = new StorageContractUpdatedEvent
+                {
+                    StoragePurchase = purchaseStatus
+                };
+            });
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+            WriteCodexEvent(e =>
+            {
+                e.StorageAvailabilityCreated = new StorageAvailabilityCreatedEvent
+                {
+                    StorageAvailability = response
+                };
+            });
+        }
+
+        private void WriteCodexEvent(Action<OverwatchCodexEvent> action)
+        {
+            WriteCodexEvent(DateTime.UtcNow, action);
+        }
+
+        private void WriteCodexEvent(DateTime utc, Action<OverwatchCodexEvent> action)
+        {
+            var e = new OverwatchCodexEvent
+            {
+                NodeIdentity = identityIndex
+            };
+
+            action(e);
+
+            if (identityIndex < 0)
+            {
+                // If we don't know our id, don't write the events yet.
+                AddToCache(utc, e);
+            }
+            else
+            {
+                e.Write(utc, writer);
+
+                // Write any events that we cached when we didn't have our id yet.
+                WriteAndClearCache();
+            }
+        }
+
+        private void AddToCache(DateTime utc, OverwatchCodexEvent e)
+        {
+            pendingEvents.Add((utc, e));
+        }
+
+        private void WriteAndClearCache()
+        {
+            if (pendingEvents.Any())
+            {
+                foreach (var pair in pendingEvents)
+                {
+                    pair.Item2.NodeIdentity = identityIndex;
+                    pair.Item2.Write(pair.Item1, writer);
+                }
+                pendingEvents.Clear();
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

@@ -0,0 +1,91 @@
+using CodexPlugin.Hooks;
+using KubernetesWorkflow;
+using Logging;
+using OverwatchTranscript;
+using Utils;
+
+namespace CodexPlugin.OverwatchSupport
+{
+    public class CodexTranscriptWriter : ICodexHooksProvider
+    {
+        private const string CodexHeaderKey = "cdx_h";
+        private readonly ILog log;
+        private readonly CodexTranscriptWriterConfig config;
+        private readonly ITranscriptWriter writer;
+        private readonly CodexLogConverter converter;
+        private readonly IdentityMap identityMap = new IdentityMap();
+        private readonly KademliaPositionFinder positionFinder = new KademliaPositionFinder();
+
+        public CodexTranscriptWriter(ILog log, CodexTranscriptWriterConfig config, ITranscriptWriter transcriptWriter)
+        {
+            this.log = log;
+            this.config = config;
+            writer = transcriptWriter;
+            converter = new CodexLogConverter(writer, config, identityMap);
+        }
+
+        public void Finalize(string outputFilepath)
+        {
+            log.Log("Finalizing Codex transcript...");
+
+            writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
+            writer.Write(outputFilepath);
+
+            log.Log("Done");
+        }
+
+        public ICodexNodeHooks CreateHooks(string nodeName)
+        {
+            nodeName = Str.Between(nodeName, "'", "'");
+            return new CodexNodeTranscriptWriter(writer, identityMap, nodeName);
+        }
+
+        public void IncludeFile(string filepath)
+        {
+            writer.IncludeArtifact(filepath);   
+        }
+
+        public void ProcessLogs(IDownloadedLog[] downloadedLogs)
+        {
+            foreach (var l in downloadedLogs)
+            {
+                log.Log("Include artifact: " + l.GetFilepath());
+                writer.IncludeArtifact(l.GetFilepath());
+
+                // Not all of these logs are necessarily Codex logs.
+                // Check, and process only the Codex ones.
+                if (IsCodexLog(l))
+                {
+                    log.Log("Processing Codex log: " + l.GetFilepath());
+                    converter.ProcessLog(l);
+                }
+            }
+        }
+
+        public void AddResult(bool success, string result)
+        {
+            writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
+            {
+                NodeIdentity = -1,
+                ScenarioFinished = new ScenarioFinishedEvent
+                {
+                    Success = success,
+                    Result = result
+                }
+            });
+        }
+
+        private OverwatchCodexHeader CreateCodexHeader()
+        {
+            return new OverwatchCodexHeader
+            {
+                Nodes = positionFinder.DeterminePositions(identityMap.Get())
+            };
+        }
+
+        private bool IsCodexLog(IDownloadedLog log)
+        {
+            return log.GetLinesContaining("Run Codex node").Any();
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs

@@ -0,0 +1,12 @@
+namespace CodexPlugin.OverwatchSupport
+{
+    public class CodexTranscriptWriterConfig
+    {
+        public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents)
+        {
+            IncludeBlockReceivedEvents = includeBlockReceivedEvents;
+        }
+
+        public bool IncludeBlockReceivedEvents { get; }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs

@@ -0,0 +1,63 @@
+namespace CodexPlugin.OverwatchSupport
+{
+    public class IdentityMap
+    {
+        private readonly List<CodexNodeIdentity> nodes = new List<CodexNodeIdentity>();
+        private readonly Dictionary<string, int> nameIndexMap = new Dictionary<string, int>();
+        private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
+
+        public void Add(string name, string peerId, string nodeId)
+        {
+            Add(new CodexNodeIdentity
+            {
+                Name = name,
+                PeerId = peerId,
+                NodeId = nodeId
+            });
+
+            nameIndexMap.Add(name, nameIndexMap.Count);
+        }
+
+        public void Add(CodexNodeIdentity identity)
+        {
+            if (string.IsNullOrWhiteSpace(identity.Name)) throw new Exception("Name required");
+            if (string.IsNullOrWhiteSpace(identity.PeerId) || identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
+            if (string.IsNullOrWhiteSpace(identity.NodeId) || identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
+
+            nodes.Add(identity);
+
+            shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
+            shortToLong.Add(CodexUtils.ToNodeIdShortId(identity.NodeId), identity.NodeId);
+        }
+
+        public CodexNodeIdentity[] Get()
+        {
+            return nodes.ToArray();
+        }
+
+        public int GetIndex(string name)
+        {
+            return nameIndexMap[name];
+        }
+
+        public CodexNodeIdentity GetId(string name)
+        {
+            return nodes.Single(n => n.Name == name);
+        }
+
+        public string ReplaceShortIds(string value)
+        {
+            var result = value;
+            foreach (var pair in shortToLong)
+            {
+                result = result.Replace(pair.Key, pair.Value);
+            }
+            return result;
+        }
+
+        public int Size
+        {
+            get { return nodes.Count; }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/KademliaPositionFinder.cs

@@ -0,0 +1,68 @@
+using System.Numerics;
+using Utils;
+using YamlDotNet.Core.Tokens;
+
+namespace CodexPlugin.OverwatchSupport
+{
+    public class KademliaPositionFinder
+    {
+        public CodexNodeIdentity[] DeterminePositions(CodexNodeIdentity[] identities)
+        {
+            var zero = identities.First();
+            var distances = CalculateDistances(zero, identities);
+
+            var maxDistance = distances.Values.Max();
+            CalculateNormalizedPositions(distances, maxDistance);
+
+            return identities;
+        }
+
+        private Dictionary<CodexNodeIdentity, BigInteger>  CalculateDistances(CodexNodeIdentity zero, CodexNodeIdentity[] identities)
+        {
+            var result = new Dictionary<CodexNodeIdentity, BigInteger>();
+            foreach (var id in identities.Skip(1))
+            {
+                result.Add(id, GetDistance(zero.NodeId, id.NodeId));
+            }
+            return result;
+        }
+
+        private BigInteger GetDistance(string id1, string id2)
+        {
+            var one = BigInteger.Parse(id1, System.Globalization.NumberStyles.HexNumber).ToByteArray();
+            var two = BigInteger.Parse(id2, System.Globalization.NumberStyles.HexNumber).ToByteArray();
+
+            var x = Xor(one, two);
+            return new BigInteger(x, isUnsigned: true);
+        }
+
+        private byte[] Xor(byte[] one, byte[] two)
+        {
+            if (one.Length != two.Length) throw new Exception("Not equal length");
+
+            var result = new byte[one.Length];
+            for (int i = 0; i < one.Length; i++)
+            {
+                uint a = one[i];
+                uint b = two[i];
+                uint c = (a ^ b);
+                result[i] = (byte)c;
+            }
+            return result;
+        }
+
+        private void CalculateNormalizedPositions(Dictionary<CodexNodeIdentity, BigInteger> distances, BigInteger maxDistance)
+        {
+            foreach (var pair in distances)
+            {
+                pair.Key.KademliaNormalizedPosition = DeterminePosition(pair.Value, maxDistance);
+            }
+        }
+
+        private float DeterminePosition(BigInteger value, BigInteger maxDistance)
+        {
+            var f = (value * 10000) / maxDistance;
+            return ((float)f) / 10000.0f;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs

@@ -0,0 +1,44 @@
+namespace CodexPlugin.OverwatchSupport.LineConverters
+{
+    public class BlockReceivedLineConverter : ILineConverter
+    {
+        public string Interest => "Received blocks from peer";
+
+        public void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent)
+        {
+            var peer = line.Attributes["peer"];
+            var blockAddresses = line.Attributes["blocks"];
+
+            SplitBlockAddresses(blockAddresses, address =>
+            {
+                addEvent(e =>
+                {
+                    e.BlockReceived = new BlockReceivedEvent
+                    {
+                        SenderPeerId = peer,
+                        BlockAddress = address
+                    };
+                });
+            });
+        }
+
+        private void SplitBlockAddresses(string blockAddresses, Action<string> onBlockAddress)
+        {
+            // Single line can contain multiple block addresses.
+            var tokens = blockAddresses.Split(",", StringSplitOptions.RemoveEmptyEntries).ToList();
+            while (tokens.Count > 0)
+            {
+                if (tokens.Count == 1)
+                {
+                    onBlockAddress(tokens[0]);
+                    return;
+                }
+
+                var blockAddress = $"{tokens[0]}, {tokens[1]}";
+                tokens.RemoveRange(0, 2);
+
+                onBlockAddress(blockAddress);
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs

@@ -0,0 +1,71 @@
+namespace CodexPlugin.OverwatchSupport.LineConverters
+{
+    public class BootstrapLineConverter : ILineConverter
+    {
+        private const string peerIdTag = "peerId: ";
+
+        public string Interest => "Starting codex node";
+
+        public void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent)
+        {
+            // "(
+            // configFile: none(InputFile),
+            // logLevel: \"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify;warn:contracts,clock;warn:serde,json,serialization\",
+            // logFormat: auto,
+            // metricsEnabled: false,
+            // metricsAddress: 127.0.0.1,
+            // metricsPort: 8008,
+            // dataDir: datadir5,
+            // circuitDir: /root/.cache/codex/circuits,
+            // listenAddrs: @[/ip4/0.0.0.0/tcp/8081],
+            // nat: 10.1.0.214,
+            // discoveryIp: 0.0.0.0,
+            // discoveryPort: 8080,
+            // netPrivKeyFile: \"key\",
+            // bootstrapNodes:
+                // @[(envelope: (publicKey: secp256k1 key (0414380858330307a4a59e8a1643512f80680dedb8c541674f40382be71c26556cd65b7e1775ec9fabfaf6d58d562b88a3c8afb969f8cc256db20e4c4c9e1f70a6),
+                // domain: \"libp2p-peer-record\",
+                // payloadType: @[3, 1],
+                // payload: @[10, 39, 0, 37, 8, 2, 18, 33, 2, 20, 56, 8, 88, 51, 3, 7, 164, 165, 158, 138, 22, 67, 81, 47, 128, 104, 13, 237, 184, 197, 65, 103, 79, 64, 56, 43, 231, 28, 38, 85, 108, 16, 254, 211, 141, 181, 6, 26, 11, 10, 9, 4, 10, 1, 0, 210, 145, 2, 31, 144],
+                // signature: 3045022100FA846871D96EDCA579990244B1C590E16B57A7BDB817908A2B580043F8A0B0280220342825DBE577E83C22CD59AA9099DE8F8DC86A38C659F60E8B9255126CF37FDE),
+                // data:
+                    // (peerId: 16Uiu2HAkvnbgwdB2NmNe1uWGJxE3Ep3sDHxNW95W2rTPs2vfxvou,
+                    // seqNo: 1721985534,
+                    // addresses: @[(address: /ip4/10.1.0.210/udp/8080)]
+                    // )
+                // )],
+            // maxPeers: 160,
+            // agentString: \"Codex\",
+            // apiBindAddress: \"0.0.0.0\",
+            // apiPort: 30035,
+            // apiCorsAllowedOrigin: none(string),
+            // repoKind: fs,
+            // storageQuota: 8589934592\'NByte,
+            // blockTtl: 1d,
+            // blockMaintenanceInterval: 10m,
+            // blockMaintenanceNumberOfBlocks: 1000,
+            // cacheSize: 0\'NByte,
+            // logFile: none(string),
+            // cmd: noCmd
+            // )"
+
+            var config = line.Attributes["config"];
+            
+            while (config.Contains(peerIdTag))
+            {
+                var openIndex = config.IndexOf(peerIdTag) + peerIdTag.Length;
+                var closeIndex = config.IndexOf(",", openIndex);
+                var bootPeerId = config.Substring(openIndex, closeIndex - openIndex);
+                config = config.Substring(closeIndex);
+
+                addEvent(e =>
+                {
+                    e.BootstrapConfig = new BootstrapConfigEvent
+                    {
+                        BootstrapPeerId = bootPeerId
+                    };
+                });
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs

@@ -0,0 +1,20 @@
+namespace CodexPlugin.OverwatchSupport.LineConverters
+{
+    public class DialSuccessfulLineConverter : ILineConverter
+    {
+        public string Interest => "Dial successful";
+
+        public void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent)
+        {
+            var peerId = line.Attributes["peerId"];
+
+            addEvent(e =>
+            {
+                e.DialSuccessful = new PeerDialSuccessfulEvent
+                {
+                    TargetPeerId = peerId
+                };
+            });
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs

@@ -0,0 +1,20 @@
+namespace CodexPlugin.OverwatchSupport.LineConverters
+{
+    public class PeerDroppedLineConverter : ILineConverter
+    {
+        public string Interest => "Dropping peer";
+
+        public void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent)
+        {
+            var peerId = line.Attributes["peer"];
+
+            addEvent(e =>
+            {
+                e.PeerDropped = new PeerDroppedEvent
+                {
+                    DroppedPeerId = peerId
+                };
+            });
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs

@@ -0,0 +1,164 @@
+using OverwatchTranscript;
+
+namespace CodexPlugin.OverwatchSupport
+{
+    [Serializable]
+    public class OverwatchCodexHeader
+    {
+        public CodexNodeIdentity[] Nodes { get; set; } = Array.Empty<CodexNodeIdentity>();
+    }
+
+    [Serializable]
+    public class OverwatchCodexEvent
+    {
+        public int NodeIdentity { get; set; } = -1;
+        public ScenarioFinishedEvent? ScenarioFinished { get; set; }
+        public NodeStartingEvent? NodeStarting { get; set; }
+        public NodeStartedEvent? NodeStarted { get; set; }
+        public NodeStoppingEvent? NodeStopping { get; set; }
+        public BootstrapConfigEvent? BootstrapConfig { get; set; }
+        public FileUploadingEvent? FileUploading { get; set; }
+        public FileUploadedEvent? FileUploaded { get; set; }
+        public FileDownloadingEvent? FileDownloading { get; set; }
+        public FileDownloadedEvent? FileDownloaded { get; set; }
+        public BlockReceivedEvent? BlockReceived { get; set; }
+        public PeerDialSuccessfulEvent? DialSuccessful { get; set; }
+        public PeerDroppedEvent? PeerDropped { get; set; }
+        public StorageContractSubmittedEvent? StorageContractSubmitted { get; set; }
+        public StorageContractUpdatedEvent? StorageContractUpdated { get; set; }
+        public StorageAvailabilityCreatedEvent? StorageAvailabilityCreated { get; set; }
+
+        public void Write(DateTime utc, ITranscriptWriter writer)
+        {
+            if (NodeIdentity == -1 && ScenarioFinished == null)
+            {
+                throw new Exception("NodeIdentity not set, and event is not ScenarioFinished.");
+            }
+            if (AllNull()) throw new Exception("No event data was set");
+
+            writer.Add(utc, this);
+        }
+
+        private bool AllNull()
+        {
+            var props = GetType()
+                .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
+                .Where(p => p.PropertyType != typeof(string)).ToArray();
+
+            return props.All(p => p.GetValue(this) == null);
+        }
+    }
+
+    [Serializable]
+    public class CodexNodeIdentity
+    {
+        public string Name { get; set; } = string.Empty;
+        public string PeerId { get; set; } = string.Empty;
+        public string NodeId { get; set; } = string.Empty;
+        public float KademliaNormalizedPosition { get; set; } = 0.0f;
+    }
+
+    #region Scenario Generated Events
+
+    [Serializable]
+    public class ScenarioFinishedEvent
+    {
+        public bool Success { get; set; }
+        public string Result { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class NodeStartingEvent
+    {
+        public string Image { get; set; } = string.Empty;
+        public string EthAddress { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class NodeStartedEvent
+    {
+    }
+
+    [Serializable]
+    public class NodeStoppingEvent
+    {
+    }
+
+    [Serializable]
+    public class BootstrapConfigEvent
+    {
+        public string BootstrapPeerId { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class FileUploadingEvent
+    {
+        public string UniqueId { get; set;} = string.Empty;
+        public long ByteSize { get; set; }
+    }
+
+    [Serializable]
+    public class FileDownloadingEvent
+    {
+        public string Cid { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class FileUploadedEvent
+    {
+        public string UniqueId { get; set; } = string.Empty;
+        public long ByteSize { get; set; }
+        public string Cid { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class FileDownloadedEvent
+    {
+        public string Cid { get; set; } = string.Empty;
+        public long ByteSize { get; set; }
+    }
+
+    [Serializable]
+    public class StorageAvailabilityCreatedEvent
+    {
+        public StorageAvailability StorageAvailability { get; set; } = null!;
+    }
+
+    [Serializable]
+    public class StorageContractUpdatedEvent
+    {
+        public StoragePurchase StoragePurchase { get; set; } = null!;
+    }
+
+    [Serializable]
+    public class StorageContractSubmittedEvent
+    {
+        public string PurchaseId { get; set; } = string.Empty;
+        public StoragePurchaseRequest PurchaseRequest { get; set; } = null!;
+    }
+
+    #endregion
+
+    #region Codex Generated Events
+
+    [Serializable]
+    public class BlockReceivedEvent
+    {
+        public string BlockAddress { get; set; } = string.Empty;
+        public string SenderPeerId { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class PeerDialSuccessfulEvent
+    {
+        public string TargetPeerId { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class PeerDroppedEvent
+    {
+        public string DroppedPeerId { get; set; } = string.Empty;
+    }
+
+    #endregion
+}

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,4 +1,5 @@
-using Logging;
+using CodexPlugin.Hooks;
+using Logging;
 using Newtonsoft.Json;
 using Utils;
 
@@ -18,19 +19,20 @@ namespace CodexPlugin
     {
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
+        private readonly ICodexNodeHooks hooks;
         private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
         private readonly DateTime contractPendingUtc = DateTime.UtcNow;
         private DateTime? contractSubmittedUtc = DateTime.UtcNow;
         private DateTime? contractStartedUtc;
         private DateTime? contractFinishedUtc;
 
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase, ICodexNodeHooks hooks)
         {
             this.log = log;
             this.codexAccess = codexAccess;
             PurchaseId = purchaseId;
             Purchase = purchase;
-
+            this.hooks = hooks;
             ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
         }
 
@@ -87,16 +89,17 @@ namespace CodexPlugin
             Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
             while (lastState != desiredState)
             {
+                Thread.Sleep(sleep);
+
                 var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
                 var statusJson = JsonConvert.SerializeObject(purchaseStatus);
                 if (purchaseStatus != null && purchaseStatus.State != lastState)
                 {
                     lastState = purchaseStatus.State;
                     log.Debug("Purchase status: " + statusJson);
+                    hooks.OnStorageContractUpdated(purchaseStatus);
                 }
 
-                Thread.Sleep(sleep);
-
                 if (lastState == "errored")
                 {
                     FrameworkAssert.Fail("Contract errored: " + statusJson);

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -3,24 +3,29 @@ using System.Text;
 
 public static class Program
 {
-    private const string OpenApiFile = "../CodexPlugin/openapi.yaml";
-    private const string ClientFile = "../CodexPlugin/obj/openapiClient.cs";
     private const string Search = "<INSERT-OPENAPI-YAML-HASH>";
-    private const string TargetFile = "ApiChecker.cs";
+    private const string CodexPluginFolderName = "CodexPlugin";
+    private const string ProjectPluginsFolderName = "ProjectPlugins";
 
     public static void Main(string[] args)
     {
         Console.WriteLine("Injecting hash of 'openapi.yaml'...");
 
-        // Force client rebuild by deleting previous artifact.
-        File.Delete(ClientFile);
+        var root = FindCodexPluginFolder();
+        Console.WriteLine("Located CodexPlugin: " + root);
+        var openApiFile = Path.Combine(root, "openapi.yaml");
+        var clientFile = Path.Combine(root, "obj", "openapiClient.cs");
+        var targetFile = Path.Combine(root, "ApiChecker.cs");
 
-        var hash = CreateHash();
+        // Force client rebuild by deleting previous artifact.
+        File.Delete(clientFile);
+
+        var hash = CreateHash(openApiFile);
         // This hash is used to verify that the Codex docker image being used is compatible
         // with the openapi.yaml being used by the Codex plugin.
         // If the openapi.yaml files don't match, an exception is thrown.
 
-        SearchAndInject(hash);
+        SearchAndInject(hash, targetFile);
 
         // This program runs as the pre-build trigger for "CodexPlugin".
         // You might be wondering why this work isn't done by a shell script.
@@ -33,9 +38,39 @@ public static class Program
         Console.WriteLine("Done!");
     }
 
-    private static string CreateHash()
+    private static string FindCodexPluginFolder()
     {
-        var file = File.ReadAllText(OpenApiFile);
+        var current = Directory.GetCurrentDirectory();
+
+        while (true)
+        {
+            var localFolders = Directory.GetDirectories(current);
+            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+            if (projectPluginsFolders.Length == 1)
+            {
+                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
+            }
+            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
+            if (codexPluginFolders.Length == 1)
+            {
+                return codexPluginFolders.Single();
+            }
+
+            var parent = Directory.GetParent(current);
+            if (parent == null)
+            {
+                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                Console.WriteLine(msg);
+                throw new Exception(msg);
+            }
+
+            current = parent.FullName;
+        }
+    }
+
+    private static string CreateHash(string openApiFile)
+    {
+        var file = File.ReadAllText(openApiFile);
         var fileBytes = Encoding.ASCII.GetBytes(file
             .Replace(Environment.NewLine, ""));
 
@@ -44,11 +79,11 @@ public static class Program
         return BitConverter.ToString(hash);
     }
 
-    private static void SearchAndInject(string hash)
+    private static void SearchAndInject(string hash, string targetFile)
     {
-        var lines = File.ReadAllLines(TargetFile);
+        var lines = File.ReadAllLines(targetFile);
         Inject(lines, hash);
-        File.WriteAllLines(TargetFile, lines);
+        File.WriteAllLines(targetFile, lines);
     }
 
     private static void Inject(string[] lines, string hash)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,9 +15,10 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
+            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
             endpoint = tools
-                .CreateHttp()
-                .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "/api/v1/");
+                .CreateHttp(address.ToString())
+                .CreateEndpoint(address, "/api/v1/");
         }
 
         public RunningContainer RunningContainer { get; }

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -71,7 +71,7 @@ namespace ContinuousTests
             var address = new Address($"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
             var baseUrl = "";
 
-            var http = tools.CreateHttp(client =>
+            var http = tools.CreateHttp(address.ToString(), client =>
             {
                 client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
             });

Tests/CodexContinuousTests/NodeRunner.cs

@@ -4,6 +4,7 @@ using Utils;
 using Core;
 using CodexPlugin;
 using KubernetesWorkflow.Types;
+using KubernetesWorkflow;
 
 namespace ContinuousTests
 {

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -1,7 +1,6 @@
 using CodexPlugin;
 using FileUtils;
 using Logging;
-using Newtonsoft.Json;
 using NUnit.Framework;
 using Utils;
 

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -1,4 +1,5 @@
-using DistTestCore;
+using CodexPlugin;
+using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
@@ -16,59 +17,56 @@ namespace CodexTests.ScalabilityTests
             [Values(100, 1000)] int fileSize
         )
         {
-            var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var hosts = StartCodex(numberOfHosts, s => s.WithName("host").WithLogLevel(CodexLogLevel.Trace));
             var file = GenerateTestFile(fileSize.MB());
-            var cid = hosts[0].UploadFile(file);
-            var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
+
+            var uploadTasks = hosts.Select(h =>
+            {
+                return Task.Run(() =>
+                {
+                    return h.UploadFile(file);
+                });
+            }).ToArray();
+
+            Task.WaitAll(uploadTasks);
+            var cid = new ContentId(uploadTasks.Select(t => t.Result.Id).Distinct().Single());
 
             var uploadLog = Ci.DownloadLog(hosts[0]);
             var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
             var blockCids = uploadLog
-                .FindLinesThatContain("Putting block into network store")
+                .FindLinesThatContain("Block Stored")
                 .Select(s =>
                 {
-                    var start = s.IndexOf("cid=") + 4;
-                    var end = s.IndexOf(" count=");
-                    var len = end - start;
-                    return s.Substring(start, len);
+                    var line = CodexLogLine.Parse(s)!;
+                    return line.Attributes["cid"];
                 })
                 .ToArray();
 
             Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
 
-            foreach (var h in hosts) h.DownloadContent(cid);
 
-            var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+
+            var client = StartCodex(s => s.WithName("client").WithLogLevel(CodexLogLevel.Trace));
             var resultFile = client.DownloadContent(cid);
             resultFile!.AssertIsEqual(file);
 
             var downloadLog = Ci.DownloadLog(client);
             var host = string.Empty;
-            var blockCidHostMap = new Dictionary<string, string>();
-            downloadLog.IterateLines(line =>
+            var blockAddressHostMap = new Dictionary<string, List<string>>();
+            downloadLog
+                .IterateLines(s =>
             {
-                if (line.Contains("peer=") && line.Contains(" len="))
-                {
-                    var start = line.IndexOf("peer=") + 5;
-                    var end = line.IndexOf(" len=");
-                    var len = end - start;
-                    host = line.Substring(start, len);
-                }
-                else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
-                {
-                    var start = line.IndexOf("cid=") + 4;
-                    var end = line.IndexOf(" count=");
-                    var len = end - start;
-                    var blockCid = line.Substring(start, len);
+                var line = CodexLogLine.Parse(s)!;
+                var peer = line.Attributes["peer"];
+                var blockAddresses = line.Attributes["blocks"];
 
-                    blockCidHostMap.Add(blockCid, host);
-                    host = string.Empty;
-                }
-            });
+                AddBlockAddresses(peer, blockAddresses, blockAddressHostMap);
 
-            var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
-            //PrintFullMap(blockCidHostMap);
-            PrintOverview(blockCidHostMap);
+            }, thatContain: "Received blocks from peer");
+
+            var totalFetched = blockAddressHostMap.Count(p => p.Value.Any());
+            PrintFullMap(blockAddressHostMap);
+            //PrintOverview(blockCidHostMap);
 
             Log("Expected number of blocks: " + expectedNumberOfBlocks);
             Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
@@ -76,13 +74,47 @@ namespace CodexTests.ScalabilityTests
             Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
         }
 
-        private void PrintOverview(Dictionary<string, string> blockCidHostMap)
+        private void AddBlockAddresses(string peer, string blockAddresses, Dictionary<string, List<string>> blockAddressHostMap)
+        {
+            // Single line can contain multiple block addresses.
+            var tokens = blockAddresses.Split(",", StringSplitOptions.RemoveEmptyEntries).ToList();
+            while (tokens.Count > 0)
+            {
+                if (tokens.Count == 1)
+                {
+                    AddBlockAddress(peer, tokens[0], blockAddressHostMap);
+                    return;
+                }
+
+                var blockAddress = $"{tokens[0]}, {tokens[1]}";
+                tokens.RemoveRange(0, 2);
+
+                AddBlockAddress(peer, blockAddress, blockAddressHostMap);
+            }
+        }
+
+        private void AddBlockAddress(string peer, string blockAddress, Dictionary<string, List<string>> blockAddressHostMap)
+        {
+            if (blockAddressHostMap.ContainsKey(blockAddress))
+            {
+                blockAddressHostMap[blockAddress].Add(peer);
+            }
+            else
+            {
+                blockAddressHostMap[blockAddress] = new List<string> { peer };
+            }
+        }
+
+        private void PrintOverview(Dictionary<string, List<string>> blockAddressHostMap)
         {
             var overview = new Dictionary<string, int>();
-            foreach (var pair in blockCidHostMap)
+            foreach (var pair in blockAddressHostMap)
             {
-                if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
-                else overview[pair.Value]++;
+                foreach (var host in pair.Value)
+                {
+                    if (!overview.ContainsKey(host)) overview.Add(host, 1);
+                    else overview[host]++;
+                }
             }
 
             Log("Blocks fetched per host:");
@@ -92,19 +124,13 @@ namespace CodexTests.ScalabilityTests
             }
         }
 
-        private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
+        private void PrintFullMap(Dictionary<string, List<string>> blockAddressHostMap)
         {
             Log("Per block, host it was fetched from:");
-            foreach (var pair in blockCidHostMap)
+            foreach (var pair in blockAddressHostMap)
             {
-                if (string.IsNullOrEmpty(pair.Value))
-                {
-                    Log($"block: {pair.Key} = Not seen");
-                }
-                else
-                {
-                    Log($"block: {pair.Key} = '{pair.Value}'");
-                }
+                var hostStr = $"[{string.Join(",", pair.Value)}]";
+                Log($"blockAddress: {pair.Key} = {hostStr}");
             }
         }
 

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
+using FileUtils;
 using GethPlugin;
 using NUnit.Framework;
 using Utils;
@@ -11,11 +12,21 @@ namespace CodexTests.BasicTests
     public class MarketplaceTests : AutoBootstrapDistTest
     {
         [Test]
-        public void MarketplaceExample()
+        [Combinatorial]
+        [CreateTranscript(nameof(MarketplaceTests), includeBlockReceivedEvents: false)]
+        public void MarketplaceExample(
+            [Values(64)] int numBlocks,
+            [Values(0)] int plusSizeKb,
+            [Values(0)] int plusSizeBytes
+        )
         {
             var hostInitialBalance = 234.TstWei();
             var clientInitialBalance = 100000.TstWei();
-            var fileSize = 10.MB();
+            var fileSize = new ByteSize(
+                numBlocks * (64 * 1024) +
+                plusSizeKb * 1024 +
+                plusSizeBytes
+            );
 
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
@@ -46,7 +57,7 @@ namespace CodexTests.BasicTests
                 host.Marketplace.MakeStorageAvailable(availability);
             }
 
-            var testFile = GenerateTestFile(fileSize);
+            var testFile = CreateFile(fileSize);
 
             var client = StartCodex(s => s
                 .WithName("Client")
@@ -55,9 +66,9 @@ namespace CodexTests.BasicTests
 
             AssertBalance(contracts, client, Is.EqualTo(clientInitialBalance));
 
-            var contentId = client.UploadFile(testFile);
+            var uploadCid = client.UploadFile(testFile);
 
-            var purchase = new StoragePurchaseRequest(contentId)
+            var purchase = new StoragePurchaseRequest(uploadCid)
             {
                 PricePerSlotPerSecond = 2.TstWei(),
                 RequiredCollateral = 10.TstWei(),
@@ -69,6 +80,18 @@ namespace CodexTests.BasicTests
             };
 
             var purchaseContract = client.Marketplace.RequestStorage(purchase);
+            
+            var contractCid = purchaseContract.ContentId;
+            Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
+
+            // Download both from client.
+            testFile.AssertIsEqual(client.DownloadContent(uploadCid));
+            testFile.AssertIsEqual(client.DownloadContent(contractCid));
+
+            // Download both from another node.
+            var downloader = StartCodex(s => s.WithName("Downloader"));
+            testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
+            testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
 
             WaitForAllSlotFilledEvents(contracts, purchase, geth);
 
@@ -85,12 +108,13 @@ namespace CodexTests.BasicTests
         }
 
         [Test]
+        [Ignore("Integrated into MarketplaceExample to speed up testing.")]
         public void CanDownloadContentFromContractCid()
         {
             var fileSize = 10.MB();
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
-            var testFile = GenerateTestFile(fileSize);
+            var testFile = CreateFile(fileSize);
 
             var client = StartCodex(s => s
                 .WithName("Client")
@@ -124,6 +148,18 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
         }
 
+        private TrackedFile CreateFile(ByteSize fileSize)
+        {
+            var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);
+
+            return GenerateTestFile(o => o
+                .Random(segmentSize)
+                .ByteRepeat(new byte[] { 0xaa }, segmentSize)
+                .Random(segmentSize)
+                .ByteRepeat(new byte[] { 0xee }, segmentSize)
+            );
+        }
+
         private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)
         {
             Time.Retry(() =>

Tests/CodexTests/BasicTests/PyramidTests.cs

@@ -0,0 +1,88 @@
+using CodexPlugin;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class PyramidTests : CodexDistTest
+    {
+        [Test]
+        [CreateTranscript(nameof(PyramidTest))]
+        public void PyramidTest()
+        {
+            var size = 5.MB();
+            var numberOfLayers = 4;
+
+            var bottomLayer = StartLayers(numberOfLayers);
+
+            var cids = UploadFiles(bottomLayer, size);
+
+            DownloadAllFilesFromEachNodeInLayer(bottomLayer, cids);
+        }
+
+        private List<ICodexNode> StartLayers(int numberOfLayers)
+        {
+            var layer = new List<ICodexNode>();
+            layer.Add(StartCodex(s => s.WithName("Top")));
+
+            for (var i = 0; i < numberOfLayers; i++)
+            {
+                var newLayer = new List<ICodexNode>();
+                foreach (var node in layer)
+                {
+                    newLayer.AddRange(StartCodex(2, s => s.WithBootstrapNode(node).WithName("Layer[" + i + "]")));
+                }
+
+                layer.Clear();
+                layer.AddRange(newLayer);
+            }
+
+            return layer;
+        }
+
+        private ContentId[] UploadFiles(List<ICodexNode> layer, ByteSize size)
+        {
+            var uploadTasks = new List<Task<ContentId>>();
+            foreach (var node in layer)
+            {
+                uploadTasks.Add(Task.Run<ContentId>(() =>
+                {
+                    var file = GenerateTestFile(size);
+                    return node.UploadFile(file);
+                }));
+            }
+
+            var cids = uploadTasks.Select(t =>
+            {
+                t.Wait();
+                return t.Result;
+            }).ToArray();
+
+            return cids;
+        }
+
+        private void DownloadAllFilesFromEachNodeInLayer(List<ICodexNode> layer, ContentId[] cids)
+        {
+            var downloadTasks = new List<Task>();
+            foreach (var node in layer)
+            {
+                downloadTasks.Add(Task.Run(() =>
+                {
+                    var dlCids = RandomUtils.Shuffled(cids);
+                    foreach (var cid in dlCids)
+                    {
+                        node.DownloadContent(cid);
+                    }
+                }));
+            }
+
+            Task.WaitAll(downloadTasks.ToArray());
+        }
+    }
+}

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -22,6 +22,24 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloadedFile);
         }
 
+        [Test]
+        [CreateTranscript(nameof(SwarmTest))]
+        public void SwarmTest()
+        {
+            var uploader = StartCodex(s => s.WithName("uploader"));
+            var downloaders = StartCodex(5, s => s.WithName("downloader"));
+
+            var file = GenerateTestFile(100.MB());
+            var cid = uploader.UploadFile(file);
+
+            var result = Parallel.ForEach(downloaders, d =>
+            {
+                d.DownloadContent(cid);
+            });
+
+            Assert.That(result.IsCompleted);
+        }
+
         [Test]
         public void DownloadingUnknownCidDoesNotCauseCrash()
         {

Tests/CodexTests/CodexDistTest.cs

@@ -1,19 +1,25 @@
 using CodexContractsPlugin;
 using CodexNetDeployer;
 using CodexPlugin;
+using CodexPlugin.OverwatchSupport;
 using CodexTests.Helpers;
 using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
 using DistTestCore.Logs;
+using Logging;
 using MetricsPlugin;
 using Newtonsoft.Json;
+using NUnit.Framework;
 using NUnit.Framework.Constraints;
+using OverwatchTranscript;
 
 namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {
+        private static readonly Dictionary<TestLifecycle, CodexTranscriptWriter> writers = new Dictionary<TestLifecycle, CodexTranscriptWriter>();
+
         public CodexDistTest()
         {
             ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
@@ -29,6 +35,18 @@ namespace CodexTests
             localBuilder.Build();
         }
 
+        protected override void LifecycleStart(TestLifecycle lifecycle)
+        {
+            base.LifecycleStart(lifecycle);
+            SetupTranscript(lifecycle);
+        }
+
+        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
+        {
+            base.LifecycleStop(lifecycle, result);
+            TeardownTranscript(lifecycle, result);
+        }
+
         public ICodexNode StartCodex()
         {
             return StartCodex(s => { });
@@ -108,5 +126,82 @@ namespace CodexTests
         protected virtual void OnCodexSetup(ICodexSetup setup)
         {
         }
+
+        private CreateTranscriptAttribute? GetTranscriptAttributeOfCurrentTest()
+        {
+            var attrs = GetCurrentTestMethodAttribute<CreateTranscriptAttribute>();
+            if (attrs.Any()) return attrs.Single();
+            return null;
+        }
+
+        private void SetupTranscript(TestLifecycle lifecycle)
+        {
+            var attr = GetTranscriptAttributeOfCurrentTest();
+            if (attr == null) return;
+
+            var config = new CodexTranscriptWriterConfig(
+                attr.IncludeBlockReceivedEvents
+            );
+
+            var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
+            var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log));
+            Ci.SetCodexHooksProvider(writer);
+            writers.Add(lifecycle, writer);
+        }
+
+        private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result)
+        {
+            var attr = GetTranscriptAttributeOfCurrentTest();
+            if (attr == null) return;
+
+            var outputFilepath = GetOutputFullPath(lifecycle, attr);
+
+            var writer = writers[lifecycle];
+            writers.Remove(lifecycle);
+
+            writer.AddResult(result.Success, result.Result);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () =>
+                {
+                    writer.ProcessLogs(lifecycle.DownloadAllLogs());
+                });
+
+                Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () =>
+                {
+                    writer.IncludeFile(lifecycle.Log.LogFile.FullFilename);
+                    writer.Finalize(outputFilepath);
+                });
+            }
+            catch (Exception ex)
+            {
+                lifecycle.Log.Error("Failure during transcript teardown: " + ex);
+            }
+        }
+
+        private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr)
+        {
+            var outputPath = Path.GetDirectoryName(lifecycle.Log.LogFile.FullFilename);
+            if (outputPath == null) throw new Exception("Logfile path is null");
+            var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.LogFile.FullFilename);
+            if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty");
+            var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename);
+            if (!outputFile.EndsWith(".owts")) outputFile += ".owts";
+            return outputFile;
+        }
+    }
+
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class CreateTranscriptAttribute : PropertyAttribute
+    {
+        public CreateTranscriptAttribute(string outputFilename, bool includeBlockReceivedEvents = true)
+        {
+            OutputFilename = outputFilename;
+            IncludeBlockReceivedEvents = includeBlockReceivedEvents;
+        }
+
+        public string OutputFilename { get; }
+        public bool IncludeBlockReceivedEvents { get; }
     }
 }

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -24,7 +24,6 @@ namespace CodexTests.UtilityTests
         private readonly List<ulong> rewardsSeen = new List<ulong>();
         private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
         private readonly List<string> receivedEvents = new List<string>();
-        private readonly List<MarketAverage> receivedAverages = new List<MarketAverage>();
  
         [Test]
         [DontDownloadLogs]
@@ -56,8 +55,6 @@ namespace CodexTests.UtilityTests
             AssertEventOccurance("Transit: New -> Started", 1);
             AssertEventOccurance("Transit: Started -> Finished", 1);
 
-            AssertMarketAverage();
-
             foreach (var r in repo.Rewards)
             {
                 var seen = rewardsSeen.Any(s => r.RoleId == s);
@@ -80,28 +77,9 @@ namespace CodexTests.UtilityTests
                 $"Event '{msg}' did not occure correct number of times.");
         }
 
-        private void AssertMarketAverage()
-        {
-            Assert.That(receivedAverages.Count, Is.EqualTo(1));
-            var a = receivedAverages.Single();
-
-            Assert.That(a.NumberOfFinished, Is.EqualTo(1));
-            Assert.That(a.TimeRangeSeconds, Is.EqualTo(5760));
-            Assert.That(a.Price, Is.EqualTo(2.0f).Within(0.1f));
-            Assert.That(a.Size, Is.EqualTo(GetMinFileSize().SizeInBytes).Within(1.0f));
-            Assert.That(a.Duration, Is.EqualTo(GetMinRequiredRequestDuration().TotalSeconds).Within(1.0f));
-            Assert.That(a.Collateral, Is.EqualTo(10.0f).Within(0.1f));
-            Assert.That(a.ProofProbability, Is.EqualTo(5.0f).Within(0.1f));
-        }
-
         private void OnCommand(string timestamp, GiveRewardsCommand call)
         {
             Log($"<API call {timestamp}>");
-            receivedAverages.AddRange(call.Averages);
-            foreach (var a in call.Averages)
-            {
-                Log("\tAverage: " + JsonConvert.SerializeObject(a));
-            }
             receivedEvents.AddRange(call.EventsOverview);
             foreach (var e in call.EventsOverview)
             {

Tests/DistTestCore/Configuration.cs

@@ -14,7 +14,7 @@ namespace DistTestCore
             kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
             logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
-            AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
+            AlwaysDownloadContainerLogs = true; // !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
         }
 
         public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath)

Tests/DistTestCore/DistTest.cs

@@ -13,7 +13,7 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
-        private const string TestNamespacePrefix = "ct-";
+        private const string TestNamespacePrefix = "cdx-";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
@@ -117,6 +117,11 @@ namespace DistTestCore
             return Get().GenerateTestFile(size, label);
         }
 
+        public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
+        {
+            return Get().GenerateTestFile(options, label);
+        }
+
         /// <summary>
         /// Any test files generated in 'action' will be deleted after it returns.
         /// This helps prevent large tests from filling up discs.
@@ -161,7 +166,7 @@ namespace DistTestCore
         {
         }
 
-        protected virtual void LifecycleStop(TestLifecycle lifecycle)
+        protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult)
         {
         }
 
@@ -213,7 +218,7 @@ namespace DistTestCore
                 WriteEndTestLog(lifecycle.Log);
 
                 IncludeLogsOnTestFailure(lifecycle);
-                LifecycleStop(lifecycle);
+                LifecycleStop(lifecycle, testResult);
                 lifecycle.DeleteAllResources();
                 lifecycles.Remove(GetCurrentTestName());
             });
@@ -257,11 +262,16 @@ namespace DistTestCore
             return CurrentTestMethodHasAttribute<DontDownloadLogsAttribute>();
         }
 
-        private bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
+        protected bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
+        {
+            return GetCurrentTestMethodAttribute<T>().Any();
+        }
+
+        protected T[] GetCurrentTestMethodAttribute<T>() where T : PropertyAttribute
         {
             // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out
-            // if the attribute is present.
+            // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to
+            // fetch the attributes of type T.
             var currentTest = TestContext.CurrentContext.Test;
             var className = currentTest.ClassName;
             var methodName = currentTest.MethodName;
@@ -269,7 +279,10 @@ namespace DistTestCore
             var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
             var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
 
-            return testMethods.Any(m => m.GetCustomAttribute<T>() != null);
+            return testMethods.Select(m => m.GetCustomAttribute<T>())
+                .Where(a => a != null)
+                .Cast<T>()
+                .ToArray();
         }
 
         private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
@@ -304,9 +317,12 @@ namespace DistTestCore
             return $"[{TestContext.CurrentContext.Test.Name}]";
         }
 
-        private string GetTestResult()
+        private DistTestResult GetTestResult()
         {
-            return TestContext.CurrentContext.Result.Outcome.Status.ToString();
+            var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed;
+            var status = TestContext.CurrentContext.Result.Outcome.Status.ToString();
+            var result = TestContext.CurrentContext.Result.Message;
+            return new DistTestResult(success, status, result ?? string.Empty);
         }
 
         private bool IsDownloadingLogsEnabled()
@@ -315,6 +331,26 @@ namespace DistTestCore
         }
     }
 
+    public class DistTestResult
+    {
+        public DistTestResult(bool success, string status, string result)
+        {
+            Success = success;
+            Status = status;
+            Result = result;
+        }
+
+        public bool Success { get; }
+        public string Status { get; }
+        public string Result { get; }
+
+        public override string ToString()
+        {
+            if (Success) return $"Passed ({Status}) ({Result})";
+            return $"Failed ({Status}) ({Result})";
+        }
+    }
+
     public static class GlobalTestFailure
     {
         public static bool HasFailed { get; set; } = false;

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -1,4 +1,4 @@
-using Core;
+using KubernetesWorkflow;
 using NUnit.Framework;
 
 namespace DistTestCore

Tests/DistTestCore/Logs/StatusLog.cs

@@ -20,9 +20,14 @@ namespace DistTestCore.Logs
             this.deployId = deployId;
         }
 
-        public void ConcludeTest(string resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
+        public void ConcludeTest(DistTestResult resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
         {
-            ConcludeTest(resultStatus, testDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), data);
+            ConcludeTest(resultStatus.Status, testDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), data);
+        }
+
+        public void ConcludeTest(string result, TimeSpan testDuration, Dictionary<string, string> data)
+        {
+            ConcludeTest(result, testDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), data);
         }
 
         public void ConcludeTest(string resultStatus, string testDuration, Dictionary<string, string> data)

Tests/DistTestCore/TestLifecycle.cs

@@ -15,6 +15,7 @@ namespace DistTestCore
         private readonly Dictionary<string, string> metadata; 
         private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
+        private readonly List<IDownloadedLog> stoppedContainerLogs = new List<IDownloadedLog>();
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
         {
@@ -52,6 +53,11 @@ namespace DistTestCore
             return entryPoint.Tools.GetFileManager().GenerateFile(size, label);
         }
 
+        public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
+        {
+            return entryPoint.Tools.GetFileManager().GenerateFile(options, label);
+        }
+
         public IFileManager GetFileManager()
         {
             return entryPoint.Tools.GetFileManager();
@@ -75,6 +81,12 @@ namespace DistTestCore
         public void OnContainersStopped(RunningPod rc)
         {
             runningContainers.Remove(rc);
+
+            stoppedContainerLogs.AddRange(rc.Containers.Select(c =>
+            {
+                if (c.StopLog == null) throw new Exception("Expected StopLog for stopped container " + c.Name);
+                return c.StopLog;
+            }));
         }
 
         public void OnContainerRecipeCreated(ContainerRecipe recipe)
@@ -93,21 +105,36 @@ namespace DistTestCore
             }
         }
 
-        public void DownloadAllLogs()
+        public IDownloadedLog[] DownloadAllLogs()
         {
             try
             {
+                var result = new List<IDownloadedLog>();
+                result.AddRange(stoppedContainerLogs);
                 foreach (var rc in runningContainers)
                 {
-                    foreach (var c in rc.Containers)
+                    if (rc.IsStopped)
                     {
-                        CoreInterface.DownloadLog(c);
+                        foreach (var c in rc.Containers)
+                        {
+                            if (c.StopLog == null) throw new Exception("No stop-log was downloaded for container.");
+                            result.Add(c.StopLog);
+                        }
+                    }
+                    else
+                    {
+                        foreach (var c in rc.Containers)
+                        {
+                            result.Add(CoreInterface.DownloadLog(c));
+                        }
                     }
                 }
+                return result.ToArray();
             }
             catch (Exception ex)
             {
                 Log.Error("Exception during log download: " + ex);
+                return Array.Empty<IDownloadedLog>();
             }
         }
     }

Tests/FrameworkTests/FrameworkTests.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\NethereumWorkflow\NethereumWorkflow.csproj" />
+    <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
   </ItemGroup>

Tests/FrameworkTests/OverwatchTranscript/TranscriptLargeTest.cs

@@ -0,0 +1,97 @@
+using Logging;
+using NUnit.Framework;
+using OverwatchTranscript;
+
+namespace FrameworkTests.OverwatchTranscript
+{
+    [TestFixture]
+    public class TranscriptLargeTests
+    {
+        private const int NumberOfThreads = 10;
+        private const int NumberOfEventsPerThread = 1000000;
+        private const string TranscriptFilename = "testtranscriptlarge.owts";
+        private ITranscriptWriter writer = null!;
+
+        [Test]
+        [Ignore("Takes about 25 minutes to run.")]
+        public void MillionsOfEvents()
+        {
+            var log = new FileLog(nameof(MillionsOfEvents));
+            writer = Transcript.NewWriter(log);
+
+            Stopwatch.Measure(log, "Generate", () =>
+            {
+                var tasks = new List<Task>();
+                for (var i = 0; i < NumberOfThreads; i++)
+                {
+                    tasks.Add(RunGeneratorThread());
+                }
+
+                Task.WaitAll(tasks.ToArray());
+            });
+
+            Stopwatch.Measure(log, "Write", () =>
+            {
+                writer.Write(TranscriptFilename);
+            });
+
+            Stopwatch.Measure(log, "Read", () =>
+            {
+                ReadTranscript();
+            });
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private Task RunGeneratorThread()
+        {
+            return Task.Run(() =>
+            {
+                try
+                {
+                    var remaining = NumberOfEventsPerThread;
+                    while (remaining > 0)
+                    {
+                        writer.Add(DateTime.UtcNow, new MyEvent
+                        {
+                            EventData = Guid.NewGuid().ToString()
+                        });
+                        remaining--;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Assert.Fail("exception in thread: " + ex);
+                }
+            });
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var expectedNumberOfEvents = NumberOfThreads * NumberOfEventsPerThread;
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(expectedNumberOfEvents));
+
+            var counter = 0;
+            var current = DateTime.MinValue;
+            reader.AddEventHandler<MyEvent>(e =>
+            {
+                counter++;
+                if (e.Moment.Utc < current)
+                {
+                    Assert.Fail("Event has moment BEFORE previous one.");
+                }
+                current = e.Moment.Utc;
+            });
+
+            var run = true;
+            while (run)
+            {
+                run = reader.Next();
+            }
+
+            reader.Close();
+        }
+    }
+}

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -0,0 +1,135 @@
+using Logging;
+using NUnit.Framework;
+using OverwatchTranscript;
+
+namespace FrameworkTests.OverwatchTranscript
+{
+    [TestFixture]
+    public class TranscriptTests
+    {
+        private const string TranscriptFilename = "testtranscript.owts";
+        private const string HeaderKey = "testHeader";
+        private const string HeaderData = "abcdef";
+        private const string EventData0 = "12345";
+        private const string EventData1 = "678";
+        private const string EventData2 = "90";
+        private const string EventData3 = "-=";
+        private readonly DateTime t0 = DateTime.UtcNow;
+        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
+        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
+
+        [Test]
+        public void WriteAndRun()
+        {
+            WriteTranscript();
+            ReadTranscript();
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private void WriteTranscript()
+        {
+            var log = new ConsoleLog();
+            var writer = Transcript.NewWriter(log);
+
+            writer.AddHeader(HeaderKey, new TestHeader
+            {
+                HeaderData = HeaderData
+            });
+
+            writer.Add(t0, new MyEvent
+            {
+                EventData = EventData0
+            });
+            writer.Add(t2, new MyEvent
+            {
+                EventData = EventData3
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData1
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData2
+            });
+
+            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
+
+            writer.Write(TranscriptFilename);
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var header = reader.GetHeader<TestHeader>(HeaderKey);
+            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
+            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
+
+            var moments = new List<ActivateMoment>();
+            var events = new List<ActivateEvent<MyEvent>>();
+            reader.AddMomentHandler(moments.Add);
+            reader.AddEventHandler<MyEvent>(events.Add);
+
+
+            Assert.That(moments.Count, Is.EqualTo(0));
+            Assert.That(events.Count, Is.EqualTo(0));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(1));
+            Assert.That(events.Count, Is.EqualTo(1));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(2));
+            Assert.That(events.Count, Is.EqualTo(3));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
+            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
+            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
+
+            AssertEvent(events[0], utc: t0, duration: t1 - t0, index: 0, data: EventData0);
+            AssertEvent(events[1], utc: t1, duration: t2 - t1, index: 1, data: EventData1);
+            AssertEvent(events[2], utc: t1, duration: t2 - t1, index: 1, data: EventData2);
+            AssertEvent(events[3], utc: t2, duration: null, index: 2, data: EventData3);
+
+            reader.Close();
+        }
+
+        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
+        {
+            Assert.That(m.Utc, Is.EqualTo(utc));
+            Assert.That(m.Duration, Is.EqualTo(duration));
+            Assert.That(m.Index, Is.EqualTo(index));
+        }
+
+        private void AssertEvent(ActivateEvent<MyEvent> e, DateTime utc, TimeSpan? duration, int index, string data)
+        {
+            Assert.That(e.Moment.Utc, Is.EqualTo(utc));
+            Assert.That(e.Moment.Duration, Is.EqualTo(duration));
+            Assert.That(e.Moment.Index, Is.EqualTo(index));
+            Assert.That(e.Payload.EventData, Is.EqualTo(data));
+        }
+    }
+
+    public class TestHeader
+    {
+        public string HeaderData { get; set; } = string.Empty;
+    }
+
+    public class MyEvent
+    {
+        public string EventData { get; set; } = string.Empty;
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -0,0 +1,193 @@
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using static FrameworkTests.Utils.RunLengthEncodingTests;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingRunTests
+    {
+        [Test]
+        [Combinatorial]
+        public void RunIncludes(
+            [Values(0, 1, 2, 3)] int start,
+            [Values(1, 2, 3, 4)] int length)
+        {
+            var run = new Run(start, length);
+
+            var shouldInclude = Enumerable.Range(start, length).ToArray();
+            var shouldExclude = new int[]
+            {
+                shouldInclude.Min() - 1,
+                shouldInclude.Max() + 1
+            };
+
+            foreach (var incl in shouldInclude)
+            {
+                Assert.That(run.Includes(incl));
+            }
+            foreach (var excl in shouldExclude)
+            {
+                Assert.That(!run.Includes(excl));
+            }
+        }
+
+        [Test]
+        public void RunExpandToInclude()
+        {
+            var run = new Run(2, 3);
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(run.ExpandToInclude(1), Is.False);
+            Assert.That(run.ExpandToInclude(2), Is.False);
+            Assert.That(run.ExpandToInclude(4), Is.False);
+            Assert.That(run.ExpandToInclude(6), Is.False);
+
+            Assert.That(run.ExpandToInclude(5), Is.True);
+            Assert.That(run.Includes(5));
+            Assert.That(!run.Includes(6));
+        }
+
+        [Test]
+        public void RunCanUnsetLastIndex()
+        {
+            var run = new Run(0, 3);
+            Assert.That(run.Includes(2));
+            var update = run.Unset(2);
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void RunCanSplit()
+        {
+            var run = new Run(0, 6); // 0, 1, 2, 3, 4, 5
+            var update = run.Unset(2);
+
+            Assert.That(run.Start, Is.EqualTo(0));
+            Assert.That(run.Length, Is.EqualTo(2)); // 0, 1
+            Assert.That(!run.Includes(2));
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+
+            Assert.That(!update.NewRuns[0].Includes(2));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(3));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(3)); // 3, 4, 5
+            Assert.That(!update.NewRuns[0].Includes(6));
+        }
+
+        [Test]
+        public void RunReplacesSelfWhenUnsetFirstIndex()
+        {
+            var run = new Run(0, 5);
+            var update = run.Unset(0);
+
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
+        [Test]
+        public void CanIterateIndices()
+        {
+            var run = new Run(2, 4);
+            var seen = new List<int>();
+            run.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
+        }
+    }
+
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public bool ExpandToInclude(int index)
+        {
+            if (index == (Start + Length))
+            {
+                Length++;
+                return true;
+            }
+            return false;
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
+                }
+                return new RunUpdate(
+                    newRuns: new[] { new Run(Start + 1, Length - 1) },
+                    removeRuns: new[] { this }
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
+        }
+        
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -0,0 +1,320 @@
+using Logging;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using NuGet.Frameworks;
+using NUnit.Framework;
+using System.Collections.Concurrent;
+using System.Numerics;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingTests
+    {
+        private readonly Random random = new Random();
+
+        [Test]
+        public void EmptySet()
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < 1000; i++)
+            {
+                Assert.That(set.IsSet(i), Is.False);
+            }
+
+            var calls = 0;
+            set.Iterate(i => calls++);
+            Assert.That(calls, Is.EqualTo(0));
+        }
+
+        [Test]
+        public void SetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+
+            Assert.That(set.IsSet(index), Is.True);
+        }
+
+        [Test]
+        public void UnsetsIndex()
+        {
+            var set = new IndexSet();
+            var index = 1234;
+            set.Set(index);
+            set.Unset(index);
+
+            Assert.That(set.IsSet(index), Is.False);
+        }
+
+        [Test]
+        public void RandomIndices()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            AssertEqual(set, indices);
+        }
+
+        [Test]
+        public void RandomRunLengthEncoding()
+        {
+            var indices = GenerateRandomIndices();
+            var set = new IndexSet(indices);
+
+            var encoded = set.RunLengthEncoded();
+            var decoded = IndexSet.FromRunLengthEncoded(encoded);
+
+            AssertEqual(decoded, indices);
+        }
+
+        [Test]
+        public void RunLengthEncoding()
+        {
+            var indices = new[] { 0, 1, 2, 4, 6, 7 };
+            var set = new IndexSet(indices);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                0, 3,
+                4, 1,
+                6, 2
+            }, encoded);
+        }
+
+        [Test]
+        public void RunLengthDecoding()
+        {
+            var encoded = new[]
+            {
+                2, 4, // 2, 3, 4, 5
+                7, 1, // 7
+                9, 2  // 9, 10
+            };
+
+            var set = IndexSet.FromRunLengthEncoded(encoded);
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(new[]
+            {
+                2, 3, 4, 5,
+                7,
+                9, 10
+            }, seen);
+        }
+
+        [Test]
+        public void SetIndexBeforeRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void SetIndexAfterRun()
+        {
+            var set = new IndexSet(new[] { 12, 13, 14 });
+            set.Set(15);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 4
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtStartOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                12, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexAtEndOfRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(14);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 3
+            }, encoded);
+        }
+
+        [Test]
+        public void UnsetIndexInRun()
+        {
+            var set = new IndexSet(new[] { 11, 12, 13, 14 });
+            set.Unset(12);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                11, 1,
+                13, 2
+            }, encoded);
+        }
+
+        private void AssertEqual(IndexSet set, int[] indices)
+        {
+            var max = indices.Max() + 1;
+            for (var i = 0; i < max; i++)
+            {
+                Assert.That(set.IsSet(i), Is.EqualTo(indices.Contains(i)));
+            }
+
+            var seen = new List<int>();
+            set.Iterate(i => seen.Add(i));
+
+            CollectionAssert.AreEqual(indices, seen);
+        }
+
+        private int[] GenerateRandomIndices()
+        {
+            var number = 1000;
+            var max = 2000;
+            var all = Enumerable.Range(0, max).ToList();
+            var result = new List<int>();
+
+            while (all.Any() && result.Count < number)
+            {
+                result.Add(all.PickOneRandom());
+            }
+
+            all.Sort();
+            return all.ToArray();
+        }
+
+        public class IndexSet
+        {
+            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+            public IndexSet()
+            {
+            }
+
+            public IndexSet(int[] indices)
+            {
+                foreach (var i in indices) Set(i);
+            }
+
+            public static IndexSet FromRunLengthEncoded(int[] rle)
+            {
+                var set = new IndexSet();
+                for (var i = 0; i < rle.Length; i += 2)
+                {
+                    var start = rle[i];
+                    var length = rle[i + 1];
+                    set.runs.Add(start, new Run(start, length));
+                }
+
+                return set;
+            }
+
+            public bool IsSet(int index)
+            {
+                if (runs.ContainsKey(index)) return true;
+
+                var run = GetRunBefore(index);
+                if (run == null) return false;
+
+                return run.Includes(index);
+            }
+
+            public void Set(int index)
+            {
+                if (runs.ContainsKey(index)) return;
+
+                var run = GetRunBefore(index);
+                if (run == null || !run.ExpandToInclude(index))
+                {
+                    CreateNewRun(index);
+                }
+            }
+
+            public void Unset(int index)
+            {
+                if (runs.ContainsKey(index))
+                {
+                    HandleUpdate(runs[index].Unset(index));
+                }
+                else
+                {
+                    var run = GetRunBefore(index);
+                    if (run == null) return;
+                    HandleUpdate(run.Unset(index));
+                }
+            }
+
+            public void Iterate(Action<int> onIndex)
+            {
+                foreach (var run in runs.Values)
+                {
+                    run.Iterate(onIndex);
+                }
+            }
+
+            public int[] RunLengthEncoded()
+            {
+                return Encode().ToArray();
+            }
+
+            private IEnumerable<int> Encode()
+            {
+                foreach (var pair in runs)
+                {
+                    yield return pair.Value.Start;
+                    yield return pair.Value.Length;
+                }
+            }
+
+            private Run? GetRunBefore(int index)
+            {
+                Run? result = null;
+                foreach (var pair in runs)
+                {
+                    if (pair.Key < index) result = pair.Value;
+                    else return result;
+                }
+                return result;
+            }
+
+            private void HandleUpdate(RunUpdate runUpdate)
+            {
+                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            }
+
+            private void CreateNewRun(int index)
+            {
+                if (runs.ContainsKey(index + 1))
+                {
+                    var length = runs[index + 1].Length + 1;
+                    runs.Add(index, new Run(index, length));
+                    runs.Remove(index + 1);
+                }
+                else
+                {
+                    runs.Add(index, new Run(index, 1));
+                }
+            }
+        }
+    }
+}

Tests/FrameworkTests/Utils/TimeTests.cs

@@ -21,5 +21,16 @@ namespace FrameworkTests.Utils
                 TimeSpan.FromSeconds(28)
             ));
         }
+
+        [Test]
+        public void Averaging()
+        {
+            var first = RollingAverage.GetNewAverage(0.0f, 1, 1.0f);
+            Assert.That(first, Is.EqualTo(1.0f));
+
+            var fifth = RollingAverage.GetNewAverage(5.0f, 5, 0.0f);
+            var expected = new[] { 5.0f, 5.0f, 5.0f, 5.0f, 0.0f }.Average();
+            Assert.That(fifth, Is.EqualTo(expected));
+        }
     }
 }

Tools/AutoClient/AutoClient.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/AutoClient/Configuration.cs

@@ -0,0 +1,48 @@
+using ArgsUniform;
+
+namespace AutoClient
+{
+    public class Configuration
+    {
+        [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
+        public string CodexHost { get; set; } = "http://localhost";
+
+        [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
+        public int CodexPort { get; set; } = 8080;
+
+        [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
+        public string DataPath { get; set; } = "datapath";
+
+        [Uniform("purchases", "np", "PURCHASES", false, "Number of concurrent purchases.")]
+        public int NumConcurrentPurchases { get; set; } = 10;
+
+        [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
+        public int ContractDurationMinutes { get; set; } = 60 * 6;
+
+        [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
+        public int ContractExpiryMinutes { get; set; } = 15;
+
+        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
+        public int NumHosts { get; set; } = 5;
+
+        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
+        public int HostTolerance { get; set; } = 2;
+
+        [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
+        public int Price { get; set; } = 10;
+
+        [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
+        public int RequiredCollateral { get; set; } = 1;
+
+        [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
+        public int FileSizeMb { get; set; } = 0;
+
+        public string LogPath
+        {
+            get
+            {
+                return Path.Combine(DataPath, "logs");
+            }
+        }
+    }
+}

Tools/AutoClient/ImageGenerator.cs

@@ -0,0 +1,55 @@
+using FileUtils;
+using Logging;
+using Utils;
+
+namespace AutoClient
+{
+    public interface IFileGenerator
+    {
+        Task<string> Generate();
+    }
+
+    public class ImageGenerator : IFileGenerator
+    {
+        private LogSplitter log;
+
+        public ImageGenerator(LogSplitter log)
+        {
+            this.log = log;
+        }
+
+        public async Task<string> Generate()
+        {
+            log.Log("Fetching random image from picsum.photos...");
+            var httpClient = new HttpClient();
+            var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
+
+            var filename = $"{Guid.NewGuid().ToString().ToLowerInvariant()}.jpg";
+            using var file = File.OpenWrite(filename);
+            await thing.CopyToAsync(file);
+
+            return filename;
+        }
+    }
+
+    public class RandomFileGenerator : IFileGenerator
+    {
+        private readonly ByteSize size;
+        private readonly FileManager fileManager;
+
+        public RandomFileGenerator(Configuration config, ILog log)
+        {
+            size = config.FileSizeMb.MB();
+            fileManager = new FileManager(log, config.DataPath);
+        }
+
+        public Task<string> Generate()
+        {
+            return Task.Run(() =>
+            {
+                var file = fileManager.GenerateFile(size);
+                return file.Filename;
+            });
+        }
+    }
+}

Tools/AutoClient/Program.cs

@@ -0,0 +1,92 @@
+using ArgsUniform;
+using AutoClient;
+using CodexOpenApi;
+using Core;
+using Logging;
+using Utils;
+
+public static class Program
+{
+    public static async Task Main(string[] args)
+    {
+        var cts = new CancellationTokenSource();
+        var cancellationToken = cts.Token;
+        Console.CancelKeyPress += (sender, args) => cts.Cancel();
+
+        var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
+        var config = uniformArgs.Parse(true);
+
+        if (config.NumConcurrentPurchases < 1)
+        {
+            throw new Exception("Number of concurrent purchases must be > 0");
+        }
+
+        var log = new LogSplitter(
+            new FileLog(Path.Combine(config.LogPath, "autoclient")),
+            new ConsoleLog()
+        );
+
+        var address = new Address(
+            host: config.CodexHost,
+            port: config.CodexPort
+        );
+
+        log.Log($"Start. Address: {address}");
+
+        var generator = CreateGenerator(config, log);
+
+        var client = new HttpClient();
+        var codex = new CodexApi(client);
+        codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+
+        await CheckCodex(codex, log);
+
+        var purchasers = new List<Purchaser>();
+        for (var i = 0; i < config.NumConcurrentPurchases; i++)
+        {
+            purchasers.Add(
+                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
+            );
+        }
+
+        var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
+        foreach (var purchaser in purchasers) 
+        {
+            purchaser.Start();
+            await Task.Delay(delayPerPurchaser);
+        }
+
+        cancellationToken.WaitHandle.WaitOne();
+
+        log.Log("Done.");
+    }
+
+    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    {
+        if (config.FileSizeMb > 0)
+        {
+            return new RandomFileGenerator(config, log);
+        }
+        return new ImageGenerator(log);
+    }
+
+    private static async Task CheckCodex(CodexApi codex, ILog log)
+    {
+        log.Log("Checking Codex...");
+        try
+        {
+            var info = await codex.GetDebugInfoAsync();
+            if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id");
+        }
+        catch (Exception ex)
+        {
+            log.Log($"Codex not OK: {ex}");
+            throw;
+        }
+    }
+
+    private static void PrintHelp()
+    {
+        Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
+    }
+}

Tools/AutoClient/Purchaser.cs

@@ -0,0 +1,166 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class Purchaser
+    {
+        private readonly ILog log;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly CodexApi codex;
+        private readonly Configuration config;
+        private readonly IFileGenerator generator;
+        private readonly CancellationToken ct;
+
+        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
+        {
+            this.log = log;
+            this.client = client;
+            this.address = address;
+            this.codex = codex;
+            this.config = config;
+            this.generator = generator;
+            this.ct = ct;
+        }
+
+        public void Start()
+        {
+            Task.Run(Worker);
+        }
+
+        private async Task Worker()
+        {
+            while (!ct.IsCancellationRequested)
+            {
+                var pid = await StartNewPurchase();
+                await WaitTillFinished(pid);
+            }
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            var cid = await UploadFile(file);
+            return await RequestStorage(cid);
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await generator.Generate();
+        }
+
+        private async Task<ContentId> UploadFile(string filename)
+        {
+            // Copied from CodexNode :/
+            using var fileStream = File.OpenRead(filename);
+
+            log.Log($"Uploading file {filename}...");
+            var response = await codex.UploadAsync(fileStream, ct);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            log.Log($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> RequestStorage(ContentId cid)
+        {
+            log.Log("Requesting storage for " + cid.Id);
+            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = config.RequiredCollateral.ToString(),
+                Duration = (config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = config.NumHosts,
+                Reward = config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = config.HostTolerance
+            }, ct);
+
+            log.Log("Purchase ID: " + result);
+
+            return result;
+        }
+
+        private async Task<string?> GetPurchaseState(string pid)
+        {
+            try
+            {
+                // openapi still don't match code.
+                var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+                if (string.IsNullOrEmpty(str)) return null;
+                var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+                log.Log($"Purchase {pid} is {sp.State}");
+                if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
+                return sp.State;
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            log.Log("Waiting...");
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (true)
+                {
+                    var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
+                    if (string.IsNullOrEmpty(status))
+                    {
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                    }
+                    else
+                    {
+                        if (status.Contains("cancel") ||
+                            status.Contains("error") ||
+                            status.Contains("finished"))
+                        {
+                            return;
+                        }
+                        if (status.Contains("started"))
+                        {
+                            await FixedDurationDelay();
+                        }
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, ct);
+        }
+    }
+}

Tools/AutoClient/README.md

@@ -0,0 +1,34 @@
+# Codex auto-client
+
+This thing will generate files, upload them, and purchase storage for them in an endless loop.
+
+Can generate random images or random data of a specified size.
+
+## How to run
+
+- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
+
+## Configuration options
+Options can be configured via CLI option or environment variable.
+
+| CLI option              | Environment variable | Description                                                                                                         |
+|-------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------|
+| "--codex-host"          | "CODEXHOST"          | Codex Host address. (default 'http://localhost')                                                                    |
+| "--codex-port"          | "CODEXPORT"          | port number of Codex API. (8080 by default)                                                                         |
+| "--datapath"            | "DATAPATH"           | Root path where all data files will be saved.                                                                       |
+| "--purchases"           | "PURCHASES"          | Number of concurrent purchases.                                                                                     |
+| "--contract-duration"   | "CONTRACTDURATION"   | contract duration in minutes. (default 6 hours)                                                                     |
+| "--contract-expiry"     | "CONTRACTEXPIRY"     | contract expiry in minutes. (default 15 minutes)                                                                    |
+| "--num-hosts"           | "NUMHOSTS"           | Number of hosts for contract. (default 5)                                                                           |
+| "--num-hosts-tolerance" | "NUMTOL"             | Number of host tolerance for contract. (default 2)                                                                  |
+| "--price"               | "PRICE"              | Price of contract. (default 10)                                                                                     |
+| "--collateral"          | "COLLATERAL"         | Required collateral. (default 1)                                                                                    |
+| "--filesizemb"          | "FILESIZEMB"         | When greater than zero, size of file generated and uploaded. When zero, random images are used instead. (default 0) |
+
+## Timing
+
+Configuration: `purchases` controls the number of concurrently running storage requests.
+Configuration: `contract-duration` controls the duration in minutes of each storage request.
+Auto-client will create a new storage request every X minutes, where X is the contract duration divided by the number of purchases.
+(Timing may start to vary when contracts fail or time out.)