Creates new overview after 3 new changes.

Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY

.dockerignore

@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/docker-autoclient.yml

@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+

.github/workflows/docker-keymaker.yml

@@ -0,0 +1,27 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-KeyMaker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      docker_repo: codexstorage/codex-keymaker
+    secrets: inherit
+

.github/workflows/docker-marketinsights.yml

@@ -0,0 +1,27 @@
+name: Docker - MarketInsights API
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/MarketInsights/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-marketinsights.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/MarketInsights/Dockerfile
+      docker_repo: codexstorage/codex-marketinsights
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -104,7 +104,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: digests-${{ matrix.target.arch }}
-          path: /tmp/digests/*
+          path: /tmp/digests
           if-no-files-found: error
           retention-days: 1
 

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath

CONTRIBUTINGPLUGINS.MD

@@ -1,10 +1,26 @@
 # Distributed System Tests for Nim-Codex
 
 ## Contributing plugins
-The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
 1. Create a library assembly in the project plugins folder.
 1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
-1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
 
 ## Constructors & Tools
 Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
@@ -20,19 +36,34 @@ Your implementation of `IProjectPlugin` must have a public constructor with a si
    }
 ```
 
-`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
-
 ## Plugin Interfaces
 The `IProjectPlugin` interface requires the implementation of two methods.
-1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
-1. `Decommission` - Should your plugin have any active system resources, free them in this method.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
 
 There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
-1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
-1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
 
 ## Core Interface
-Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
 ```C#
    public static class CoreInterfaceExtensions
    {
@@ -48,12 +79,14 @@ Any functionality your plugin wants to expose to code which uses the framework w
    }
 ```
 
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
 While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
 
 ## Deploying, Wrapping, and Starting
 When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
-1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
-1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
 1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
 
 Here's an example:
@@ -69,8 +102,8 @@ public static class CoreInterfaceExtensions
 
       public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
       {
-         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
-         // represents the API of your project.
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
       }
 
       public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
@@ -82,5 +115,44 @@ public static class CoreInterfaceExtensions
    }
 ```
 
-The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
 

CONTRIBUTINGTESTS.MD

@@ -21,7 +21,7 @@ Do you want to write some tests using this distributed test setup? Great! Here's
    1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
 1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
 ### Continuous tests
-1. Add new code files to `Tests/CodexContinousTests/Tests`
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
 1. Inherrit from `ContinuousTest`
 1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
 1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)

Framework/ArgsUniform/ArgsUniform.cs

@@ -4,9 +4,8 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
+        private readonly Assigner<T> assigner;
         private readonly Action printAppInfo;
-        private readonly object? defaultsProvider;
-        private readonly IEnv.IEnv env;
         private readonly string[] args;
         private const int cliStart = 8;
         private const int shortStart = 38;
@@ -31,9 +30,9 @@ namespace ArgsUniform
         public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
             this.printAppInfo = printAppInfo;
-            this.defaultsProvider = defaultsProvider;
-            this.env = env;
             this.args = args;
+
+            assigner = new Assigner<T>(env, args, defaultsProvider);
         }
 
         public T Parse(bool printResult = false)
@@ -42,7 +41,7 @@ namespace ArgsUniform
             {
                 printAppInfo();
                 PrintHelp();
-                throw new Exception();
+                Environment.Exit(0);
             }
 
             var result = Activator.CreateInstance<T>();
@@ -53,18 +52,16 @@ namespace ArgsUniform
                 var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
                 if (attr != null)
                 {
-                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
                     {
-                        {
-                            missingRequired.Add(uniformProperty);
-                        }
+                        missingRequired.Add(uniformProperty);
                     }
                 }
             }
 
             if (missingRequired.Any())
             {
-                PrintResults(result, uniformProperties);
+                PrintResults(printResult,result, uniformProperties);
                 Print("");
                 foreach (var missing in missingRequired)
                 {
@@ -75,37 +72,39 @@ namespace ArgsUniform
                 }
 
                 PrintHelp();
-                throw new ArgumentException("Unable to assemble all required arguments");
+                Environment.Exit(1);
             }
 
-            if (printResult)
-            {
-                PrintResults(result, uniformProperties);
-            }
+            PrintResults(printResult, result, uniformProperties);
 
             return result;
         }
 
-        private void PrintResults(T result, PropertyInfo[] uniformProperties)
-        {
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
-            }
-            Print("");
-        }
-
         public void PrintHelp()
         {
             Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
-            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
-            foreach (var attr in attrs)
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
+            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            foreach (var prop in props)
             {
-                var a = attr!;
-                var optional = !a.Required ? " *" : "";
-                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+                var a = prop.GetCustomAttribute<UniformAttribute>();
+                if (a != null)
+                {
+                    var optional = !a.Required ? " (optional)" : "";
+                    var def = assigner.DescribeDefaultFor(prop);
+                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
+                }
+            }
+            Print("");
+        }
+
+        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
+        {
+            if (!printResult) return;
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
             }
             Print("");
         }
@@ -115,7 +114,7 @@ namespace ArgsUniform
             Console.WriteLine(msg);
         }
 
-        private void PrintAligned(string cli, string s, string env, string desc)
+        private void PrintAligned(string cli, string s, string env, string desc, string def)
         {
             Console.CursorLeft = cliStart;
             Console.Write(cli);
@@ -124,132 +123,8 @@ namespace ArgsUniform
             Console.CursorLeft = envStart;
             Console.Write(env);
             Console.CursorLeft = descStart;
-            Console.Write(desc + Environment.NewLine);
-        }
-
-        private object GetDefaultValue(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-
-        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var currentValue = uniformProperty.GetValue(result);
-            var isEmptryString = (currentValue as string) == string.Empty;
-            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
-
-            if (defaultsProvider == null) return false;
-
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return false;
-
-            var value = defaultProperty.GetValue(defaultsProvider);
-            if (value != null)
-            {
-                return Assign(result, uniformProperty, value);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-                    
-                    throw new NotSupportedException();
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
+            Console.Write(desc + " ");
+            Console.Write(def + Environment.NewLine);
         }
     }
 }

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/ArgsUniform/Assigner.cs

@@ -0,0 +1,186 @@
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class Assigner<T>
+    {
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private readonly object? defaultsProvider;
+
+        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
+        {
+            this.env = env;
+            this.args = args;
+            this.defaultsProvider = defaultsProvider;
+        }
+
+        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        public string DescribeDefaultFor(PropertyInfo property)
+        {
+            var obj = Activator.CreateInstance<T>();
+            var defaultValue = GetDefaultValue(obj, property);
+            if (defaultValue == null) return "";
+            if (defaultValue is string str)
+            {
+                return "\"" + str + "\"";
+            }
+            return defaultValue.ToString() ?? string.Empty;
+        }
+
+        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
+        {
+            // Get value from object's static initializer if it's there.
+            var currentValue = uniformProperty.GetValue(result);
+            if (currentValue != null) return currentValue;
+
+            // Get value from defaults-provider object if it's there.
+            if (defaultsProvider == null) return null;
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return null;
+            return defaultProperty.GetValue(defaultsProvider);
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var defaultValue = GetDefaultValue(result, uniformProperty);
+            var isEmptryString = (defaultValue as string) == string.Empty;
+            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
+            {
+                return Assign(result, uniformProperty, defaultValue);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
+
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+
+        private static object GetDefaultValueForType(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+    }
+}

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Core/CoreInterface.cs

@@ -30,11 +30,7 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var file = entryPoint.Tools.GetLog().CreateSubfile();
-            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
-            var logHandler = new LogDownloadHandler(container.Name, file);
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return logHandler.DownloadLog();
+            return workflow.DownloadContainerLog(container, tailLines);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/Endpoint.cs

@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}

Framework/Core/EntryPoint.cs

@@ -38,10 +38,14 @@ namespace Core
             return new CoreInterface(this);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
         }
 
         internal T GetPlugin<T>() where T : IProjectPlugin

Framework/Core/Http.cs

@@ -1,215 +1,84 @@
 using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
 using Utils;
 
 namespace Core
 {
     public interface IHttp
     {
-        string HttpGetString(string route);
-        T HttpGetJson<T>(string route);
-        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
-        string HttpPostJson<TRequest>(string route, TRequest body);
-        TResponse HttpPostString<TResponse>(string route, string body);
-        string HttpPostStream(string route, Stream stream);
-        Stream HttpGetStream(string route);
-        T Deserialize<T>(string json);
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
     internal class Http : IHttp
     {
-        private static readonly object httpLock = new object();
+        private static object lockLock = new object();
+        private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
-        private readonly Address address;
-        private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
-        private readonly string? logAlias;
+        private readonly string id;
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        internal Http(string id, ILog log, ITimeSet timeSet)
+            : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
+            this.id = id;
             this.log = log;
             this.timeSet = timeSet;
-            this.address = address;
-            this.baseUrl = baseUrl;
             this.onClientCreated = onClientCreated;
-            this.logAlias = logAlias;
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
 
-        public string HttpGetString(string route)
+        public T OnClient<T>(Func<HttpClient, T> action)
         {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
+        {
+            var client = GetClient();
+
             return LockRetry(() =>
             {
-                return GetString(route);
-            }, $"HTTP-GET:{route}");
+                return action(client);
+            }, retry);
         }
 
-        public T HttpGetJson<T>(string route)
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
         {
-            return LockRetry(() =>
-            {
-                var json = GetString(route);
-                return Deserialize<T>(json);
-            }, $"HTTP-GET:{route}");
+            return new Endpoint(log, this, address, baseUrl, logAlias);
         }
 
-        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        private string GetDescription()
         {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                var json = Time.Wait(response.Content.ReadAsStringAsync());
-                if (!response.IsSuccessStatusCode)
-                {
-                    throw new HttpRequestException(json);
-                }
-                Log(GetUrl() + route, json);
-                return Deserialize<TResponse>(json);
-            }, $"HTTP-POST-JSON: {route}");
+            return DebugStack.GetCallerName(skipFrames: 2);
         }
 
-        public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public TResponse HttpPostString<TResponse>(string route, string body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJsonString(route, body);
-                if (response == null) throw new Exception("Received no response.");
-                var result = Deserialize<TResponse>(response);
-                if (result == null) throw new Exception("Failed to deserialize response");
-                return result;
-            }, $"HTTO-POST-JSON: {route}");
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return LockRetry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STREAM: {route}");
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return LockRetry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                return Time.Wait(client.GetStreamAsync(url));
-            }, $"HTTP-GET-STREAM: {route}");
-        }
-
-        public T Deserialize<T>(string json)
-        {
-            var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
-            {
-                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
-                {
-                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
-                    {
-                        errors.Add($"""
-                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
-                                    Path: {args.ErrorContext.Path}
-                                    Error: {args.ErrorContext.Error.Message}
-                                    """);
-                        args.ErrorContext.Handled = true;
-                    }
-                }
-            });
-            if (errors.Count > 0)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
-            }
-            else if (deserialized == null)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
-            }
-            return deserialized;
-        }
-
-        private string GetString(string route)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, "");
-            var result = Time.Wait(client.GetAsync(url));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            using var content = JsonContent.Create(body);
-            Log(url, JsonConvert.SerializeObject(body));
-            return Time.Wait(client.PostAsync(url, content));
-        }
-
-        private string PostJsonString(string route, string body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, body);
-            var content = new StringContent(body);
-            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-            var result = Time.Wait(client.PostAsync(url, content));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private string GetUrl()
-        {
-            return $"{address.Host}:{address.Port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            if (logAlias != null)
-            {
-                log.Debug($"({logAlias})({url}) = '{message}'", 3);
-            }
-            else
-            {
-                log.Debug($"({url}) = '{message}'", 3);
-            }
-        }
-
-        private T LockRetry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, Retry retry)
         {
+            var httpLock = GetLock();
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+                return retry.Run(operation);
+            }
+        }
+
+        private object GetLock()
+        {
+            lock (lockLock) // I had to.
+            {
+                if (!httpLocks.ContainsKey(id)) httpLocks.Add(id, new object());
+                return httpLocks[id];
             }
         }
 

Framework/Core/LogDownloadHandler.cs

@@ -1,28 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace Core
-{
-    internal class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly LogFile log;
-
-        internal LogDownloadHandler(string description, LogFile log)
-        {
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        internal IDownloadedLog DownloadLog()
-        {
-            return new DownloadedLog(log);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}

Framework/Core/PluginManager.cs

@@ -34,12 +34,12 @@
             return metadata;
         }
 
-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
             foreach (var pair in pairs)
             {
                 pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
             }
         }
 

Framework/Core/PluginTools.cs

@@ -1,13 +1,18 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
-using Utils;
 
 namespace Core
 {
     public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
     {
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+        ITimeSet TimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
     }
 
     public interface IWorkflowTool
@@ -22,9 +27,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp(string id);
     }
 
     public interface IFileTool
@@ -34,37 +39,38 @@ namespace Core
 
     internal class PluginTools : IPluginTools
     {
-        private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
-        private ILog log;
+        private readonly LogPrefixer log;
 
         internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
         {
-            this.log = log;
+            this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            this.timeSet = timeSet;
+            TimeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
+        public ITimeSet TimeSet { get; }
+
         public void ApplyLogPrefix(string prefix)
         {
-            log = new LogPrefixer(log, prefix);
+            log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+            return CreateHttp(id, onClientCreated, TimeSet);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
+            return new Http(id, log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
+        public IHttp CreateHttp(string id)
         {
-            return new Http(log, timeSet, address, baseUrl, logAlias);
+            return new Http(id, log, TimeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
@@ -72,9 +78,9 @@ namespace Core
             return workflowCreator.CreateWorkflow(namespaceOverride);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
             if (deleteTrackedFiles) fileManager.DeleteAllFiles();
         }
 

Framework/Core/TimeSet.cs

@@ -2,10 +2,31 @@
 {
     public interface ITimeSet
     {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
         TimeSpan HttpCallTimeout();
-        int HttpMaxNumberOfRetries();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
         TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
         TimeSpan K8sOperationTimeout();
     }
 
@@ -13,12 +34,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(3);
+            return TimeSpan.FromMinutes(2);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 3;
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -26,7 +47,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -41,27 +62,27 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromHours(2);
+            return TimeSpan.FromMinutes(30);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 1;
+            return TimeSpan.FromHours(2.2);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(2);
+            return TimeSpan.FromSeconds(20);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(15);
+            return TimeSpan.FromHours(1);
         }
     }
 }

Framework/DiscordRewards/CheckConfig.cs

@@ -13,9 +13,9 @@ namespace DiscordRewards
     public enum CheckType
     {
         Uninitialized,
-        FilledSlot,
-        FinishedSlot,
-        PostedContract,
-        StartedContract,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
     }
 }

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,6 +3,13 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || EventsOverview.Any();
+        }
     }
 
     public class RewardUsersCommand
@@ -10,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/DiscordRewards/RewardRepo.cs

@@ -1,53 +1,53 @@
-using Utils;
-
-namespace DiscordRewards
+namespace DiscordRewards
 {
     public class RewardRepo
     {
         private static string Tag => RewardConfig.UsernameTag;
 
-        public RewardConfig[] Rewards { get; } = new RewardConfig[]
-        {
-            // Filled any slot
-            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
-            {
-                Type = CheckType.FilledSlot
-            }),
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];
 
-            // Finished any slot
-            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot
-            }),
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),
 
-            // Finished a sizable slot
-            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
-            }),
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),
 
-            // Posted any contract
-            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
-            {
-                Type = CheckType.PostedContract
-            }),
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),
 
-            // Started any contract
-            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.StartedContract
-            }),
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),
 
-            // Started a sizable contract
-            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot,
-                MinNumberOfHosts = 4,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
-            })
-        };
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
     }
 }

Framework/FileUtils/FileManager.cs

@@ -7,27 +7,35 @@ namespace FileUtils
     {
         TrackedFile CreateEmptyFile(string label = "");
         TrackedFile GenerateFile(ByteSize size, string label = "");
+        TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
         void DeleteAllFiles();
         void ScopedFiles(Action action);
         T ScopedFiles<T>(Func<T> action);
     }
 
+    public interface IGenerateOption
+    {
+        IGenerateOption Random(ByteSize size);
+        IGenerateOption StringRepeat(string str, ByteSize size);
+        IGenerateOption StringRepeat(string str, int times);
+        IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
+        IGenerateOption ByteRepeat(byte[] bytes, int times);
+    }
+
     public class FileManager : IFileManager
     {
-        public const int ChunkSize = 1024 * 1024 * 100;
-        private static NumberSource folderNumberSource = new NumberSource(0);
-        private readonly Random random = new Random();
+        private static readonly NumberSource folderNumberSource = new NumberSource(0);
         private readonly ILog log;
-        private readonly string rootFolder;
         private readonly string folder;
         private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
 
+        public const int ChunkSize = 1024 * 1024 * 100;
+
         public FileManager(ILog log, string rootFolder)
         {
             folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
 
             this.log = log;
-            this.rootFolder = rootFolder;
         }
 
         public TrackedFile CreateEmptyFile(string label = "")
@@ -41,11 +49,16 @@ namespace FileUtils
             return result;
         }
 
-        public TrackedFile GenerateFile(ByteSize size, string label)
+        public TrackedFile GenerateFile(ByteSize size, string label = "")
+        {
+            return GenerateFile(o => o.Random(size), label);
+        }
+
+        public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
         {
             var sw = Stopwatch.Begin(log);
-            var result = GenerateRandomFile(size, label);
-            sw.End($"Generated file '{result.Describe()}'.");
+            var result = RunGenerators(options, label);
+            sw.End($"Generated file {result.Describe()}.");
             return result;
         }
 
@@ -57,16 +70,27 @@ namespace FileUtils
         public void ScopedFiles(Action action)
         {
             PushFileSet();
-            action();
-            PopFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         public T ScopedFiles<T>(Func<T> action)
         {
             PushFileSet();
-            var result = action();
-            PopFileSet();
-            return result;
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         private void PushFileSet()
@@ -89,26 +113,35 @@ namespace FileUtils
             if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
         }
 
-        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
         {
             var result = CreateEmptyFile(label);
-            CheckSpaceAvailable(result, size);
+            var generators = GetGenerators(options);
+            CheckSpaceAvailable(result, generators.GetRequiredSpace());
 
-            GenerateFileBytes(result, size);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            generators.Run(stream);
             return result;
         }
 
-        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
+        {
+            var result = new GeneratorCollection();
+            options(result);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
         {
             var file = new FileInfo(testFile.Filename);
             var drive = new DriveInfo(file.Directory!.Root.FullName);
 
             var spaceAvailable = drive.TotalFreeSpace;
 
-            if (spaceAvailable < size.SizeInBytes)
+            if (spaceAvailable < requiredSize)
             {
                 var msg = $"Not enough disk space. " +
-                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(requiredSize)} required. " +
                     $"{Formatter.FormatByteSize(spaceAvailable)} available.";
 
                 log.Log(msg);
@@ -116,34 +149,6 @@ namespace FileUtils
             }
         }
 
-        private void GenerateFileBytes(TrackedFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            int chunkSize = ChunkSize;
-            while (bytesLeft > 0)
-            {
-                try
-                {
-                    var length = Math.Min(bytesLeft, chunkSize);
-                    AppendRandomBytesToFile(result, length);
-                    bytesLeft -= length;
-                }
-                catch
-                {
-                    chunkSize = chunkSize / 2;
-                    if (chunkSize < 1024) throw;
-                }
-            }
-        }
-
-        private void AppendRandomBytesToFile(TrackedFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
         private void EnsureDirectory()
         {
             Directory.CreateDirectory(folder);

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/FileUtils/Generators.cs

@@ -0,0 +1,141 @@
+using System.Text;
+using Utils;
+
+namespace FileUtils
+{
+    public class GeneratorCollection : IGenerateOption
+    {
+        private readonly List<IGenerator> generators = new List<IGenerator>();
+
+        public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
+        {
+            var times = size.SizeInBytes / bytes.Length;
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption ByteRepeat(byte[] bytes, int times)
+        {
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption Random(ByteSize size)
+        {
+            generators.Add(new RandomGenerator(size));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, ByteSize size)
+        {
+            var times = size.SizeInBytes / str.Length;
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, int times)
+        {
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public void Run(FileStream file)
+        {
+            foreach (var generator in generators)
+            {
+                generator.Generate(file);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return generators.Sum(g => g.GetRequiredSpace());
+        }
+    }
+
+    public interface IGenerator
+    {
+        void Generate(FileStream file);
+        long GetRequiredSpace();
+    }
+
+    public class ByteRepeater : IGenerator
+    {
+        private readonly byte[] bytes;
+        private readonly long times;
+
+        public ByteRepeater(byte[] bytes, long times)
+        {
+            this.bytes = bytes;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            for (var i = 0; i < times; i++)
+            {
+                file.Write(bytes, 0, bytes.Length);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return bytes.Length * times;
+        }
+    }
+
+    public class StringRepeater : IGenerator
+    {
+        private readonly string str;
+        private readonly long times;
+
+        public StringRepeater(string str, long times)
+        {
+            this.str = str;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            using var writer = new StreamWriter(file);
+            for (var i = 0; i < times; i++)
+            {
+                writer.Write(str);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return Encoding.ASCII.GetBytes(str).Length * times;
+        }
+    }
+
+    public class RandomGenerator : IGenerator
+    {
+        private readonly Random random = new Random();
+        private readonly ByteSize size;
+
+        public RandomGenerator(ByteSize size)
+        {
+            this.size = size;
+        }
+
+        public void Generate(FileStream file)
+        {
+            var bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var size = Math.Min(bytesLeft, FileManager.ChunkSize);
+                var bytes = new byte[size];
+                random.NextBytes(bytes);
+                file.Write(bytes, 0, bytes.Length);
+                bytesLeft -= size;
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return size.SizeInBytes;
+        }
+    }
+}

Framework/FileUtils/TrackedFile.cs

@@ -65,7 +65,7 @@ namespace FileUtils
 
                 if (readExpected == 0 && readActual == 0)
                 {
-                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
                     return;
                 }
 

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/ByteSizeExtensions.cs

@@ -1,41 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public static class ByteSizeExtensions
-    {
-        public static string ToSuffixNotation(this ByteSize b)
-        {
-            long x = 1024;
-            var map = new Dictionary<long, string>
-            {
-                { Pow(x, 4), "Ti" },
-                { Pow(x, 3), "Gi" },
-                { Pow(x, 2), "Mi" },
-                { (x), "Ki" },
-            };
-
-            var bytes = b.SizeInBytes;
-            foreach (var pair in map)
-            {
-                if (bytes > pair.Key)
-                {
-                    double bytesD = bytes;
-                    double divD = pair.Key;
-                    double numD = Math.Ceiling(bytesD / divD);
-                    var v = Convert.ToInt64(numD);
-                    return $"{v}{pair.Value}";
-                }
-            }
-
-            return $"{bytes}";
-        }
-
-        private static long Pow(long x, int v)
-        {
-            long result = 1;
-            for (var i = 0; i < v; i++) result *= x;
-            return result;
-        }
-    }
-}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,7 +11,6 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
-        private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
@@ -27,11 +26,10 @@ namespace KubernetesWorkflow
             cts = new CancellationTokenSource();
         }
 
-        public void Start(ILogHandler logHandler)
+        public void Start()
         {
             if (worker != null) throw new InvalidOperationException();
 
-            this.logHandler = logHandler;
             cts = new CancellationTokenSource();
             worker = Task.Run(Worker);
         }
@@ -50,7 +48,9 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client);
+            var result = HasContainerBeenRestarted(client);
+            if (result) DownloadCrashedContainerLogs(client);
+            return result;
         }
 
         private void Worker()
@@ -83,14 +83,16 @@ namespace KubernetesWorkflow
         private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result) log.Log("Pod crash detected for " + containerName);
+            return result;
         }
 
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
-            log.Log("Pod crash detected for " + containerName);
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            logHandler!.Log(stream);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            handler.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -1,11 +1,15 @@
 using Logging;
 
-namespace Core
+namespace KubernetesWorkflow
 {
     public interface IDownloadedLog
     {
+        string ContainerName { get; }
+
+        void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
+        string GetFilepath();
         void DeleteFile();
     }
 
@@ -13,9 +17,28 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(LogFile logFile)
+        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
         {
-            this.logFile = logFile;
+            logFile = logHandler.LogFile;
+            ContainerName = containerName;
+        }
+
+        public string ContainerName { get; }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (thatContain.All(line.Contains))
+                {
+                    action(line);
+                }
+                line = streamReader.ReadLine();
+            }
         }
 
         public string[] GetLinesContaining(string expectedString)
@@ -57,6 +80,11 @@ namespace Core
             return result.ToArray();
         }
 
+        public string GetFilepath()
+        {
+            return logFile.FullFilename;
+        }
+
         public void DeleteFile()
         {
             File.Delete(logFile.FullFilename);

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         {
             var config = GetConfig();
             UpdateHostAddress(config);
+            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
             return config;
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,6 +43,11 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container);
+        }
+
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -59,14 +64,14 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
         {
             log.Debug();
 
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
             logHandler.Log(stream);
         }
 
@@ -110,7 +115,7 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteAllNamespacesStartingWith(string prefix)
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
         {
             log.Debug();
 
@@ -119,25 +124,28 @@ namespace KubernetesWorkflow
 
             foreach (var ns in namespaces)
             {
-                DeleteNamespace(ns);
+                DeleteNamespace(ns, wait);
             }
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(K8sNamespace))
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
             }
         }
 
-        public void DeleteNamespace(string ns)
+        public void DeleteNamespace(string ns, bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(ns))
             {
                 client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
             }
         }
 
@@ -372,7 +380,6 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -528,7 +535,7 @@ namespace KubernetesWorkflow
             }
             if (set.Memory.SizeInBytes != 0)
             {
-                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
             }
             return result;
         }
@@ -701,14 +708,14 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxRetries: 6,
+                maxTimeout: TimeSpan.FromMinutes(1),
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -864,16 +871,45 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
-        private void WaitUntilDeploymentOnline(string deploymentName)
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
+        private void WaitUntilDeploymentOnline(RunningContainer container)
         {
             WaitUntil(() =>
             {
-                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
+                CheckForCrash(container);
+
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            });
+            }, nameof(WaitUntilDeploymentOnline));
+        }
+
+        private void CheckForCrash(RunningContainer container)
+        {
+            var deploymentName = container.Recipe.Name;
+            var podName = GetPodName(container);
+
+            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
+            if (podInfo == null) return;
+            if (podInfo.Status == null) return;
+            if (podInfo.Status.ContainerStatuses == null) return;
+
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result)
+            {
+                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
+                log.Error(msg);
+
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+
+                throw new Exception(msg);
+            }
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -883,7 +919,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            });
+            }, nameof(WaitUntilDeploymentOffline));
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -892,19 +928,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            });
+            }, nameof(WaitUntilPodsForDeploymentAreOffline));
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
             }
             finally
             {
-                sw.End("", 1);
+                sw.End(msg, 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningContainers runningContainers);
-        void OnContainersStopped(RunningContainers runningContainers);
+        void OnContainersStarted(RunningPod runningPod);
+        void OnContainersStopped(RunningPod runningPod);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningContainers runningContainers)
+        public void OnContainersStarted(RunningPod runningPod)
         {
         }
 
-        public void OnContainersStopped(RunningContainers runningContainers)
+        public void OnContainersStopped(RunningPod runningPod)
         {
         }
 

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Logging;
+
+namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
@@ -20,4 +22,25 @@
 
         protected abstract void ProcessLine(string line);
     }
+
+    public class WriteToFileLogHandler : LogHandler, ILogHandler
+    {
+        public WriteToFileLogHandler(ILog sourceLog, string description)
+        {
+            LogFile = sourceLog.CreateSubfile();
+
+            var msg = $"{description} -->> {LogFile.FullFilename}";
+            sourceLog.Log(msg);
+
+            LogFile.Write(msg);
+            LogFile.WriteRaw(description);
+        }
+
+        public LogFile LogFile { get; }
+
+        protected override void ProcessLine(string line)
+        {
+            LogFile.WriteRaw(line);
+        }
+    }
 }

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace KubernetesWorkflow.Recipe
 {
@@ -21,14 +22,13 @@ namespace KubernetesWorkflow.Recipe
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
             if (userData == null) return default;
-            var jobject = (JObject)userData.UserData;
-            return jobject.ToObject<T>();
+            return JsonConvert.DeserializeObject<T>(userData.UserData);
         }
 
         private static Additional ConvertToAdditional(object userData)
         {
             var typeName = GetTypeName(userData.GetType());
-            return new Additional(typeName, userData);
+            return new Additional(typeName, JsonConvert.SerializeObject(userData));
         }
 
         private static string GetTypeName(Type type)
@@ -41,13 +41,13 @@ namespace KubernetesWorkflow.Recipe
 
     public class Additional
     {
-        public Additional(string type, object userData)
+        public Additional(string type, string userData)
         {
             Type = type;
             UserData = userData;
         }
 
         public string Type { get; }
-        public object UserData { get; }
+        public string UserData { get; }
     }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,8 +2,9 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(DateTime recipeCreatedUtc, int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
+            RecipeCreatedUtc = recipeCreatedUtc;
             Number = number;
             NameOverride = nameOverride;
             Image = image;
@@ -31,6 +32,7 @@
             if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
+        public DateTime RecipeCreatedUtc { get; }
         public string Name { get; }
         public int Number { get; }
         public string? NameOverride { get; }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
+            var recipe = new ContainerRecipe(DateTime.UtcNow, containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -105,7 +105,7 @@ namespace KubernetesWorkflow.Recipe
 
         protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
         {
-            var size = 10.MB().ToSuffixNotation();
+            var size = 10.MB().SizeInBytes.ToString();
             volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
         }
 
@@ -114,7 +114,7 @@ namespace KubernetesWorkflow.Recipe
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                resourceQuantity: volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.SizeInBytes.ToString()));
         }
 
         protected void Additional(object userData)

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,8 +1,5 @@
-using k8s;
-using k8s.Models;
-using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Recipe;
 using KubernetesWorkflow.Types;
-using Newtonsoft.Json;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,16 +9,17 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningContainers containers);
+        PodInfo GetPodInfo(RunningPod pod);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers containers, bool waitTillStopped);
-        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
+        void Stop(RunningPod pod, bool waitTillStopped);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace();
-        void DeleteNamespacesStartingWith(string namespacePrefix);
+        void DeleteNamespace(bool wait);
+        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
     }
 
     public class StartupWorkflow : IStartupWorkflow
@@ -45,12 +46,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,25 +61,36 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningContainers(startupConfig, startResult, containers);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return rc;
+                return new FutureContainers(rc, this);
+            });
+        }
+
+        public void WaitUntilOnline(RunningPod rc)
+        {
+            K8s(controller =>
+            {
+                foreach (var c in rc.Containers)
+                {
+                    controller.WaitUntilOnline(c);
+                }
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningContainers containers)
+        public PodInfo GetPodInfo(RunningPod pod)
         {
-            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -86,21 +98,43 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
+        public void Stop(RunningPod runningPod, bool waitTillStopped)
+        {
+            if (runningPod.IsStopped) return;
+            foreach (var c in runningPod.Containers)
+            {
+                c.StopLog = DownloadContainerLog(c);
+            }
+            runningPod.IsStopped = true;
+
+            K8s(controller =>
+            {
+                controller.Stop(runningPod.StartResult, waitTillStopped);
+            });
+
+            cluster.Configuration.Hooks.OnContainersStopped(runningPod);
+        }
+
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
         {
+            var msg = $"Downloading container log for '{container.Name}'";
+            log.Log(msg);
+            var logHandler = new WriteToFileLogHandler(log, msg);
+
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
+
+            return new DownloadedLog(logHandler, container.Name);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -111,19 +145,19 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteNamespace();
+                controller.DeleteNamespace(wait);
             });
         }
 
-        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
             });
         }
 
@@ -136,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(name, r, addresses);
+                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow.Types
+{
+    public class FutureContainers
+    {
+        private readonly RunningPod runningPod;
+        private readonly StartupWorkflow workflow;
+
+        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
+        {
+            this.runningPod = runningPod;
+            this.workflow = workflow;
+        }
+
+        public RunningPod WaitForOnline()
+        {
+            workflow.WaitUntilOnline(runningPod);
+            return runningPod;
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,27 +7,29 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            Id = id;
             Name = name;
             Recipe = recipe;
             Addresses = addresses;
         }
 
+        public string Id { get; }
         public string Name { get; }
         public ContainerRecipe Recipe { get; }
         public ContainerAddress[] Addresses { get; }
+        public IDownloadedLog? StopLog { get; internal set; }
 
         [JsonIgnore]
-        public RunningContainers RunningContainers { get; internal set; } = null!;
+        public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 
@@ -50,5 +52,21 @@ namespace KubernetesWorkflow.Types
             }
             throw new Exception("Running location not known.");
         }
+
+        public override string ToString()
+        {
+            return Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningContainer container &&
+                   Id == container.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -1,44 +0,0 @@
-using Newtonsoft.Json;
-
-namespace KubernetesWorkflow.Types
-{
-    public class RunningContainers
-    {
-        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
-        {
-            StartupConfig = startupConfig;
-            StartResult = startResult;
-            Containers = containers;
-
-            foreach (var c in containers) c.RunningContainers = this;
-        }
-
-        public StartupConfig StartupConfig { get; }
-        public StartResult StartResult { get; }
-        public RunningContainer[] Containers { get; }
-
-        [JsonIgnore]
-        public string Name
-        {
-            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
-        }
-
-        public string Describe()
-        {
-            return string.Join(",", Containers.Select(c => c.Name));
-        }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
-    }
-}

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -0,0 +1,61 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningPod
+    {
+        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        {
+            Id = id;
+            StartupConfig = startupConfig;
+            StartResult = startResult;
+            Containers = containers;
+
+            foreach (var c in containers) c.RunningPod = this;
+        }
+
+        public string Id { get; }
+        public StartupConfig StartupConfig { get; }
+        public StartResult StartResult { get; }
+        public RunningContainer[] Containers { get; }
+
+        [JsonIgnore]
+        public string Name
+        {
+            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
+        }
+
+        [JsonIgnore]
+        public bool IsStopped { get; internal set; }
+
+        public string Describe()
+        {
+            return string.Join(",", Containers.Select(c => c.Name));
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningPod pod &&
+                   Id == pod.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+
+        public override string ToString()
+        {
+            if (IsStopped) return Name + " (*)";
+            return Name;
+        }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static string Describe(this RunningPod[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
+}

Framework/Logging/BaseLog.cs

@@ -77,7 +77,7 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
-        private string ApplyReplacements(string str)
+        protected string ApplyReplacements(string str)
         {
             if (IsDebug) return str;
             foreach (var replacement in replacements)

Framework/Logging/ConsoleLog.cs

@@ -9,7 +9,7 @@
 
         public override void Log(string message)
         {
-            Console.WriteLine(message);
+            Console.WriteLine(ApplyReplacements(message));
         }
     }
 }

Framework/Logging/LogPrefixer.cs

@@ -3,14 +3,21 @@
     public class LogPrefixer : ILog
     {
         private readonly ILog backingLog;
-        private readonly string prefix;
+
+        public LogPrefixer(ILog backingLog)
+        {
+            this.backingLog = backingLog;
+        }
 
         public LogPrefixer(ILog backingLog, string prefix)
         {
             this.backingLog = backingLog;
-            this.prefix = prefix;
+            Prefix = prefix;
         }
 
+        public string Prefix { get; set; } = string.Empty;
+
+
         public LogFile CreateSubfile(string ext = "log")
         {
             return backingLog.CreateSubfile(ext);
@@ -18,17 +25,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(prefix + message, skipFrames);
+            backingLog.Debug(Prefix + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(prefix + message);
+            backingLog.Error(Prefix + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(prefix + message);
+            backingLog.Log(Prefix + message);
         }
 
         public void AddStringReplace(string from, string to)

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/BlockUtils/BlockCache.cs

@@ -4,7 +4,7 @@
     {
         public delegate void CacheClearedEvent();
 
-        private const int MaxEntries = 1024;
+        private const int MaxEntries = 1024 * 1024 * 5;
         private readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
 
         public event CacheClearedEvent? OnCacheCleared;
@@ -35,5 +35,7 @@
             if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null;
             return value;
         }
+
+        public int Size { get { return entries.Count; } }
     }
 }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,22 +18,41 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
+        public BlockTimeEntry Get(ulong blockNumber)
+        {
+            bounds.Initialize();
+            var b = cache.Get(blockNumber);
+            if (b != null) return b;
+            return GetBlock(blockNumber);
+        }
+
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector);
+            return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
         }
 
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector);
+            return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;
+        }
+
+        private ulong Log(Func<ulong> operation)
+        {
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
+            var result = operation();
+            sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
+
+            return result;
         }
 
         private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func<DateTime, BlockTimeEntry, bool> isWhatIwant)
@@ -70,7 +89,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var next = GetBlock(entry.BlockNumber + 1);
             return
-                entry.Utc < target &&
+                entry.Utc <= target &&
                 next.Utc > target;
         }
 
@@ -78,7 +97,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var previous = GetBlock(entry.BlockNumber - 1);
             return
-                entry.Utc > target &&
+                entry.Utc >= target &&
                 previous.Utc < target;
         }
 

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -89,26 +89,9 @@ namespace NethereumWorkflow
             }
         }
 
-        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new()
         {
-            var wrapper = new Web3Wrapper(web3, log);
-            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
-
-            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
-            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
-
-            if (!fromBlock.HasValue)
-            {
-                log.Error("Failed to find lowest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-            if (!toBlock.HasValue)
-            {
-                log.Error("Failed to find highest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-
-            return GetEvents<TEvent>(address, fromBlock.Value, toBlock.Value);
+            return GetEvents<TEvent>(address, blockRange.From, blockRange.To);
         }
 
         public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
@@ -119,5 +102,32 @@ namespace NethereumWorkflow
             var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
             return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
         }
+
+        public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            if (fromBlock == null  || toBlock == null)
+            {
+                throw new Exception("Failed to convert time range to block range.");
+            }
+
+            return new BlockInterval(
+                timeRange: timeRange,
+                from: fromBlock.Value,
+                to: toBlock.Value
+            );
+        }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+            return blockTimeFinder.Get(number);
+        }
     }
 }

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>

Framework/OverwatchTranscript/ActionQueue.cs

@@ -0,0 +1,61 @@
+namespace OverwatchTranscript
+{
+    public class ActionQueue
+    {
+        // Using ConcurrentQueue<> here would make this process slower.
+        private readonly object queueLock = new object();
+        private readonly AutoResetEvent signal = new AutoResetEvent(false);
+        private List<Action> queue = new List<Action>();
+        private Task queueWorker = null!;
+        private bool stopping = false;
+
+        public void Start()
+        {
+            queueWorker = Task.Run(QueueWorker);
+        }
+
+        public int Count { get; private set; }
+
+        public void StopAndJoin()
+        {
+            stopping = true;
+            queueWorker.Wait();
+            if (queue.Count > 0) throw new Exception("not all acions handled");
+            queueWorker.Dispose();
+        }
+
+        public void Add(Action action)
+        {
+            if (stopping) throw new Exception("queue stopping");
+
+            lock (queueLock)
+            {
+                queue.Add(action);
+                Count = queue.Count;
+            }
+            signal.Set();
+        }
+
+        private void QueueWorker()
+        {
+            while (true)
+            {
+                signal.WaitOne(10);
+
+                List<Action> work = null!;
+                lock (queueLock)
+                {
+                    work = queue;
+                    queue = new List<Action>();
+                    Count = 0;
+                }
+                if (stopping && !work.Any()) return;
+                
+                foreach (var action in work)
+                {
+                    action();
+                }
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/BucketSet.cs

@@ -0,0 +1,110 @@
+using Logging;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public class BucketSet
+    {
+        private const int numberOfActiveBuckets = 10;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private readonly object _bucketLock = new object();
+        private readonly List<EventBucketWriter> fullBuckets = new List<EventBucketWriter>();
+        private readonly List<EventBucketWriter> activeBuckets = new List<EventBucketWriter>();
+        private readonly ActionQueue queue = new ActionQueue();
+        private int activeBucketIndex = 0;
+        private bool closed = false;
+        private string internalErrors = string.Empty;
+        
+        public BucketSet(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+
+            for (var i = 0; i < numberOfActiveBuckets;i++)
+            {
+                AddNewBucket();
+            }
+
+            queue.Start();
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            if (closed) throw new Exception("Buckets already closed!");
+            queue.Add(() => AddInternal(utc, payload));
+            
+            if (queue.Count > 1000)
+            {
+                Thread.Sleep(1);
+            }
+        }
+
+        public IFinalizedBucket[] FinalizeBuckets()
+        {
+            closed = true;
+            queue.StopAndJoin();
+
+            if (IsEmpty()) throw new Exception("No entries have been added.");
+            if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors);
+
+            var buckets = fullBuckets.Concat(activeBuckets).ToArray();
+            log.Debug($"Finalizing {buckets.Length} buckets...");
+
+            var finalized = new ConcurrentBag<IFinalizedBucket>();
+            var tasks = Parallel.ForEach(buckets, b => finalized.Add(b.FinalizeBucket()));
+            if (!tasks.IsCompleted) throw new Exception("Failed to finalize buckets: " + tasks);
+
+            return finalized.ToArray();
+        }
+
+        private bool IsEmpty()
+        {
+            return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
+        }
+
+        private void AddInternal(DateTime utc, object payload)
+        {
+            try
+            {
+                lock (_bucketLock)
+                {
+                    var current = activeBuckets[activeBucketIndex];
+                    current.Add(utc, payload);
+                    activeBucketIndex = (activeBucketIndex + 1) % numberOfActiveBuckets;
+
+                    if (current.IsFull)
+                    {
+                        log.Debug("Bucket is full. New bucket...");
+                        fullBuckets.Add(current);
+                        activeBuckets.Remove(current);
+                        AddNewBucket();
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                internalErrors += ex.ToString();
+                log.Error(ex.ToString());
+            }
+        }
+
+        private static int bucketSizeIndex = 0;
+        private static int[] bucketSizes = new[]
+        {
+            10000,
+            15000,
+            20000,
+        };
+
+        private void AddNewBucket()
+        {
+            lock (_bucketLock)
+            {
+                var size = bucketSizes[bucketSizeIndex];
+                bucketSizeIndex = (bucketSizeIndex + 1) % bucketSizes.Length;
+                activeBuckets.Add(new EventBucketWriter(log, Path.Combine(workingDir, Guid.NewGuid().ToString()), size));
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -0,0 +1,151 @@
+using Logging;
+using Newtonsoft.Json;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface IFinalizedBucket
+    {
+        bool IsEmpty { get; }
+        void Update();
+        DateTime? SeeTopUtc();
+        BucketTop? TakeTop();
+    }
+
+    public class BucketTop
+    {
+        public BucketTop(DateTime utc, OverwatchEvent[] events)
+        {
+            Utc = utc;
+            Events = events;
+        }
+
+        public DateTime Utc { get; }
+        public OverwatchEvent[] Events { get; }
+    }
+
+    public class EventBucketReader : IFinalizedBucket
+    {
+        private readonly string bucketFile;
+        private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
+        private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
+
+        public EventBucketReader(ILog log, string bucketFile)
+        {
+            this.bucketFile = bucketFile;
+            if (!File.Exists(bucketFile)) throw new Exception("Doesn't exist: " + bucketFile);
+
+            log.Debug("Read Bucket open: " + bucketFile);
+
+            Task.Run(ReadBucket);
+        }
+        
+        public bool IsEmpty { get; private set; }
+
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
+        public DateTime? SeeTopUtc()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryPeek(out BucketTop? top))
+            {
+                return top.Utc;
+            }
+            return null;
+        }
+
+        public BucketTop? TakeTop()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryDequeue(out BucketTop? top))
+            {
+                itemDequeued.Set();
+                return top;
+            }
+            return null;
+        }
+
+        private void ReadBucket()
+        {
+            using var file = File.OpenRead(bucketFile);
+            using var reader = new StreamReader(file);
+
+            while (true)
+            {
+                while (topQueue.Count < 5)
+                {
+                    var top = CreateNewTop(reader);
+                    if (top != null)
+                    {
+                        topQueue.Enqueue(top);
+                        itemEnqueued.Set();
+                    }
+                    else
+                    {
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
+                        return;
+                    }
+                }
+
+                itemDequeued.Reset();
+                itemDequeued.WaitOne(5000);
+            }
+        }
+
+        private void UpdateIsEmpty()
+        {
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
+            {
+                File.Delete(bucketFile);
+                IsEmpty = true;
+            }
+        }
+
+        private EventBucketEntry? nextEntry = null;
+        private BucketTop? CreateNewTop(StreamReader reader)
+        {
+            if (nextEntry == null)
+            {
+                nextEntry = ReadEntry(reader);
+                if (nextEntry == null) return null;
+            }
+
+            var topEntry = nextEntry;
+            var entries = new List<EventBucketEntry>
+            {
+                topEntry
+            };
+
+            nextEntry = ReadEntry(reader);
+            while (nextEntry != null && nextEntry.Utc == topEntry.Utc)
+            {
+                entries.Add(nextEntry);
+                nextEntry = ReadEntry(reader);
+            }
+
+            return new BucketTop(topEntry.Utc, entries.Select(e => e.Event).ToArray());
+        }
+
+        private EventBucketEntry? ReadEntry(StreamReader reader)
+        {
+            var line = reader.ReadLine();
+            if (string.IsNullOrEmpty(line)) return null;
+            return JsonConvert.DeserializeObject<EventBucketEntry>(line);
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketWriter.cs

@@ -0,0 +1,113 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class EventBucketWriter
+    {
+        private const int MaxBuffer = 1000;
+
+        private readonly object _lock = new object();
+        private bool closed = false;
+        private readonly ILog log;
+        private readonly string bucketFile;
+        private readonly int maxCount;
+        private readonly List<EventBucketEntry> buffer = new List<EventBucketEntry>();
+
+        public EventBucketWriter(ILog log, string bucketFile, int maxCount)
+        {
+            this.log = log;
+            this.bucketFile = bucketFile;
+            this.maxCount = maxCount;
+            if (File.Exists(bucketFile)) throw new Exception("Already exists");
+
+            log.Debug("Write Bucket open: " + bucketFile);
+        }
+
+        public int Count { get; private set; }
+        public bool IsFull { get; private set; }
+
+        public void Add(DateTime utc, object payload)
+        {
+            lock (_lock)
+            {
+                if (closed) throw new Exception("Already closed");
+                AddToBuffer(utc, payload);
+                BufferToFile(emptyBuffer: false);
+            }
+        }
+
+        public IFinalizedBucket FinalizeBucket()
+        {
+            lock (_lock)
+            {
+                closed = true;
+                BufferToFile(emptyBuffer: true);
+                SortFileByTimestamps();
+            }
+            log.Debug($"Finalized bucket with {Count} entries");
+            return new EventBucketReader(log, bucketFile);
+        }
+
+        public override string ToString()
+        {
+            return $"EventBucket: " + Count;
+        }
+
+        private void AddToBuffer(DateTime utc, object payload)
+        {  
+            var typeName = payload.GetType().FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+            if (utc == default) throw new Exception("DateTimeUtc not set");
+
+            var entry = new EventBucketEntry
+            {
+                Utc = utc,
+                Event = new OverwatchEvent
+                {
+                    Type = typeName,
+                    Payload = Json.Serialize(payload)
+                }
+            };
+
+            Count++;
+            IsFull = Count > maxCount;
+
+            buffer.Add(entry);
+        }
+
+        private void BufferToFile(bool emptyBuffer)
+        {
+            if (emptyBuffer || buffer.Count > MaxBuffer)
+            {
+                using var file = File.Open(bucketFile, FileMode.Append);
+                using var writer = new StreamWriter(file);
+                foreach (var entry in buffer)
+                {
+                    writer.WriteLine(Json.Serialize(entry));
+                }
+                log.Debug($"Bucket wrote {buffer.Count} entries to file.");
+                buffer.Clear();
+            }
+        }
+
+        private void SortFileByTimestamps()
+        {
+            var lines = File.ReadAllLines(bucketFile);
+            var entries = lines.Select(Json.Deserialize<EventBucketEntry>)
+                .Cast<EventBucketEntry>()
+                .OrderBy(e => e.Utc)
+                .ToArray();
+
+            File.Delete(bucketFile);
+            File.WriteAllLines(bucketFile, entries.Select(e => Json.Serialize(e)));
+        }
+    }
+
+    [Serializable]
+    public class EventBucketEntry
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent Event { get; set; } = new();
+    }
+}

Framework/OverwatchTranscript/Json.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System.Globalization;
+
+namespace OverwatchTranscript
+{
+    public static class Json
+    {
+        private static JsonSerializerSettings settings = new JsonSerializerSettings
+        {
+            Formatting = Formatting.None,
+            NullValueHandling = NullValueHandling.Ignore,
+            Culture = CultureInfo.InvariantCulture,
+            DateFormatHandling = DateFormatHandling.IsoDateFormat,
+            FloatFormatHandling = FloatFormatHandling.Symbol
+        };
+
+        public static string Serialize(object obj, Formatting formatting = Formatting.None)
+        {
+            return JsonConvert.SerializeObject(obj, formatting, settings);
+        }
+
+        public static T Deserialize<T>(string json)
+        {
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}

Framework/OverwatchTranscript/Model.cs

@@ -0,0 +1,56 @@
+namespace OverwatchTranscript
+{
+    [Serializable]
+    public class OverwatchTranscript
+    {
+        public OverwatchHeader Header { get; set; } = new();
+        public OverwatchMomentReference[] MomentReferences { get; set; } = Array.Empty<OverwatchMomentReference>();
+    }
+
+    [Serializable]
+    public class OverwatchMomentReference
+    {
+        public string MomentsFile { get; set; } = string.Empty;
+        public int NumberOfMoments { get; set; }
+        public int NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeader
+    {
+        public OverwatchCommonHeader Common { get; set; } = new();
+        public OverwatchHeaderEntry[] Entries { get; set; } = Array.Empty<OverwatchHeaderEntry>();
+    }
+
+    [Serializable]
+    public class OverwatchCommonHeader
+    {
+        public long NumberOfMoments { get; set; }
+        public long NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeaderEntry
+    {
+        public string Key { get; set; } = string.Empty;
+        public string Value { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class OverwatchMoment
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent[] Events { get; set; } = Array.Empty<OverwatchEvent>();
+    }
+
+    [Serializable]
+    public class OverwatchEvent
+    {
+        public string Type { get; set; } = string.Empty;
+        public string Payload { get; set; } = string.Empty;
+    }
+}

Framework/OverwatchTranscript/MomentReader.cs

@@ -0,0 +1,104 @@
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReader
+    {
+        private readonly OverwatchTranscript model;
+        private readonly string workingDir;
+        private int referenceIndex = 0;
+        private int momentsRead = 0;
+        private OpenReference currentRef;
+
+        public MomentReader(OverwatchTranscript model, string workingDir)
+        {
+            this.model = model;
+            this.workingDir = workingDir;
+
+            currentRef = CreateOpenReference();
+        }
+
+        public OverwatchMoment? Next()
+        {
+            if (referenceIndex >= model.MomentReferences.Length) return null;
+
+            var moment = currentRef.ReadNext();
+            if (moment == null)
+            {
+                Close();
+
+                // This reference file ran out.
+                // The number of moments read should match exactly the number of moments
+                // describe in the reference. If not, error:
+                var expected = model.MomentReferences[referenceIndex].NumberOfMoments;
+                if (momentsRead != expected)
+                {
+                    throw new Exception("Number of moments read from referenced file does not match number of moments value in model. " +
+                        $"Reads: { momentsRead} - model.MomentReferences[{referenceIndex}].NumberOfMoment: {expected}"); 
+                }
+
+                referenceIndex++;
+                if (referenceIndex < model.MomentReferences.Length)
+                {
+                    // Proceed to next reference file.
+                    currentRef = CreateOpenReference();
+                    momentsRead = 0;
+                    return Next();
+                }
+                else
+                {
+                    // That was the last one.
+                    return null;
+                }
+            }
+            else
+            {
+                momentsRead++;
+                return moment;
+            }
+        }
+
+        public void Close()
+        {
+            if (currentRef != null)
+            {
+                currentRef.Close();
+                currentRef = null!;
+            }
+        }
+
+        private OpenReference CreateOpenReference()
+        {
+            var filepath = Path.Combine(workingDir, model.MomentReferences[referenceIndex].MomentsFile);
+            return new OpenReference(filepath);
+        }
+
+        private class OpenReference
+        {
+            private readonly FileStream file;
+            private readonly StreamReader reader;
+
+            public OpenReference(string filePath)
+            {
+                file = File.OpenRead(filePath);
+                reader = new StreamReader(file);
+            }
+
+            public OverwatchMoment? ReadNext()
+            {
+                var line = reader.ReadLine();
+                if (string.IsNullOrEmpty(line)) return null;
+                return JsonConvert.DeserializeObject<OverwatchMoment>(line);
+            }
+
+            public void Close()
+            {
+                reader.Close();
+                file.Close();
+
+                reader.Dispose();
+                file.Dispose();
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -0,0 +1,153 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReferenceBuilder
+    {
+        private const int MaxMomentsPerReference = 10000;
+        private readonly ILog log;
+        private readonly string workingDir;
+
+        public MomentReferenceBuilder(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+        }
+
+        public OverwatchMomentReference[] Build(IFinalizedBucket[] finalizedBuckets)
+        {
+            var result = new List<OverwatchMomentReference>();
+            var currentBuilder = new Builder(log, workingDir);
+
+            var buckets = finalizedBuckets.ToList();
+            log.Debug($"Building references for {buckets.Count} buckets.");
+            while (buckets.Any())
+            {
+                foreach (var b in buckets) b.Update();
+
+                buckets.RemoveAll(b => b.IsEmpty);
+                if (!buckets.Any()) break;
+
+                var earliestUtc = GetEarliestUtc(buckets);
+                if (earliestUtc == null) continue;
+                
+                var tops = CollectAllTopsForUtc(earliestUtc.Value, buckets);
+                var moment = ConvertTopsToMoment(tops);
+                currentBuilder.Add(moment);
+                if (currentBuilder.NumberOfMoments == MaxMomentsPerReference)
+                {
+                    result.Add(currentBuilder.Build());
+                    currentBuilder = new Builder(log, workingDir);
+                }
+            }
+
+            if (currentBuilder.NumberOfMoments > 0)
+            {
+                result.Add(currentBuilder.Build());
+            }
+
+            return result.ToArray();
+        }
+
+        private OverwatchMoment ConvertTopsToMoment(List<BucketTop> tops)
+        {
+            var discintUtc = tops.Select(e => e.Utc).Distinct().ToArray();
+            if (discintUtc.Length != 1) throw new Exception("UTC mixing in moment construction.");
+
+            return new OverwatchMoment
+            {
+                Utc = tops[0].Utc,
+                Events = tops.SelectMany(e => e.Events).ToArray()
+            };
+        }
+
+        private List<BucketTop> CollectAllTopsForUtc(DateTime earliestUtc, List<IFinalizedBucket> buckets)
+        {
+            var result = new List<BucketTop>();
+
+            foreach (var bucket in buckets)
+            {
+                if (bucket.IsEmpty) continue;
+
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) continue;
+
+                if (utc.Value == earliestUtc)
+                {
+                    var top = bucket.TakeTop();
+                    if (top == null) throw new Exception("top was null after top utc was not");
+                    result.Add(top);
+                }
+            }
+
+            return result;
+        }
+
+        private DateTime? GetEarliestUtc(List<IFinalizedBucket> buckets)
+        {
+            var earliest = DateTime.MaxValue;
+            foreach (var bucket in buckets)
+            {
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) return null;
+
+                if (utc.Value < earliest) earliest = utc.Value;
+            }
+            return earliest;
+        }
+
+        public class Builder
+        {
+            private readonly ILog log;
+            private readonly string workingDir;
+            private OverwatchMomentReference reference;
+            private readonly ActionQueue queue = new ActionQueue();
+
+            public Builder(ILog log, string workingDir)
+            {
+                reference = new OverwatchMomentReference
+                {
+                    MomentsFile = Guid.NewGuid().ToString(),
+                    EarliestUtc = DateTime.MaxValue,
+                    LatestUtc = DateTime.MinValue,
+                    NumberOfEvents = 0,
+                    NumberOfMoments = 0,
+                };
+                this.log = log;
+                this.workingDir = workingDir;
+                queue.Start();
+            }
+
+            public int NumberOfMoments => reference.NumberOfMoments;
+
+            public void Add(OverwatchMoment moment)
+            {
+                if (moment.Utc < reference.EarliestUtc) reference.EarliestUtc = moment.Utc;
+                if (moment.Utc > reference.LatestUtc) reference.LatestUtc = moment.Utc;
+                reference.NumberOfMoments++;
+                reference.NumberOfEvents += moment.Events.Length;
+
+                var filePath = Path.Combine(workingDir, reference.MomentsFile);
+
+                queue.Add(() =>
+                {
+                    File.AppendAllLines(filePath, new[]
+                    {
+                        Json.Serialize(moment)
+                    });
+                });
+            }
+
+            public OverwatchMomentReference Build()
+            {
+                queue.StopAndJoin();
+
+                log.Debug($"Created reference with {reference.NumberOfMoments} moments and {reference.NumberOfEvents} events...");
+                var result = reference;
+                reference = null!;
+                return result;
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/OverwatchTranscript/Transcript.cs

@@ -0,0 +1,23 @@
+using Logging;
+
+namespace OverwatchTranscript
+{
+    public static class Transcript
+    {
+        public static ITranscriptWriter NewWriter(ILog log)
+        {
+            log = new LogPrefixer(log, "(TranscriptWriter) ");
+            return new TranscriptWriter(log, NewWorkDir());
+        }
+
+        public static ITranscriptReader NewReader(string transcriptFile)
+        {
+            return new TranscriptReader(NewWorkDir(), transcriptFile);
+        }
+
+        private static string NewWorkDir()
+        {
+            return Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        }
+    }
+}

Framework/OverwatchTranscript/TranscriptConstants.cs

@@ -0,0 +1,8 @@
+namespace OverwatchTranscript
+{
+    public static class TranscriptConstants
+    {
+        public const string TranscriptFilename = "transcript.json";
+        public const string ArtifactFolderName = "artifacts";
+    }
+}

Framework/OverwatchTranscript/TranscriptReader.cs

@@ -0,0 +1,264 @@
+using Newtonsoft.Json;
+using System.IO;
+using System;
+using System.IO.Compression;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptReader
+    {
+        OverwatchCommonHeader Header { get; }
+        T GetHeader<T>(string key);
+        void AddMomentHandler(Action<ActivateMoment> handler);
+        void AddEventHandler<T>(Action<ActivateEvent<T>> handler);
+        bool Next();
+        void Close();
+    }
+
+    public class TranscriptReader : ITranscriptReader
+    {
+        private readonly object handlersLock = new object();
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly List<Action<ActivateMoment>> momentHandlers = new List<Action<ActivateMoment>>();
+        private readonly Dictionary<string, List<Action<ActivateMoment, string>>> eventHandlers = new Dictionary<string, List<Action<ActivateMoment, string>>>();
+        private readonly string workingDir;
+        private readonly OverwatchTranscript model;
+        private bool closed;
+        private long momentCounter;
+        private readonly ConcurrentQueue<OverwatchMoment> queue = new ConcurrentQueue<OverwatchMoment>();
+        private readonly Task queueFiller;
+
+        public TranscriptReader(string workingDir, string inputFilename)
+        {
+            closed = false;
+            this.workingDir = workingDir;
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+
+            model = LoadModel(inputFilename);
+
+            queueFiller = Task.Run(() => FillQueue(model, workingDir));
+        }
+
+        public OverwatchCommonHeader Header
+        {
+            get
+            {
+                CheckClosed();
+                return model.Header.Common;
+            }
+        }
+
+        public T GetHeader<T>(string key)
+        {
+            CheckClosed();
+            var value = model.Header.Entries.First(e => e.Key == key).Value;
+            return JsonConvert.DeserializeObject<T>(value)!;
+        }
+
+        public void AddMomentHandler(Action<ActivateMoment> handler)
+        {
+            CheckClosed();
+            lock (handlersLock)
+            {
+                momentHandlers.Add(handler);
+            }
+        }
+
+        public void AddEventHandler<T>(Action<ActivateEvent<T>> handler)
+        {
+            CheckClosed();
+
+            var typeName = typeof(T).FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+
+            lock (handlersLock)
+            {
+                if (eventHandlers.ContainsKey(typeName))
+                {
+                    eventHandlers[typeName].Add(CreateEventAction(handler));
+                }
+                else
+                {
+                    eventHandlers.Add(typeName, new List<Action<ActivateMoment, string>>
+                    {
+                        CreateEventAction(handler)
+                    });
+                }
+            }
+        }
+
+        private readonly object nextLock = new object();
+        private OverwatchMoment? moment = null;
+        private OverwatchMoment? next = null;
+
+        public bool Next()
+        {
+            CheckClosed();
+
+            OverwatchMoment? m = null;
+            TimeSpan? duration = null;
+            lock (nextLock)
+            {
+                if (next == null)
+                {
+                    if (!queue.TryDequeue(out moment)) return false;
+                    queue.TryDequeue(out next);
+                }
+                else
+                {
+                    moment = next;
+                    next = null;
+                    queue.TryDequeue(out next);
+                }
+
+                m = moment;
+                duration = GetMomentDuration();
+            }
+
+            ActivateMoment(moment, duration);
+
+            return true;
+        }
+
+        public void Close()
+        {
+            CheckClosed();
+            closed = true;
+
+            queueFiller.Wait();
+
+            Directory.Delete(workingDir, true);
+        }
+
+        private Action<ActivateMoment, string> CreateEventAction<T>(Action<ActivateEvent<T>> handler)
+        {
+            return (m, s) =>
+            {
+                handler(new ActivateEvent<T>(m, JsonConvert.DeserializeObject<T>(s)!));
+            };
+        }
+
+        private void FillQueue(OverwatchTranscript model, string workingDir)
+        {
+            var reader = new MomentReader(model, workingDir);
+
+            while (true)
+            {
+                if (closed)
+                {
+                    reader.Close();
+                    return;
+                }
+
+                while (queue.Count < 10)
+                {
+                    var moment = reader.Next();
+                    if (moment == null)
+                    {
+                        reader.Close();
+                        return;
+                    }
+                    queue.Enqueue(moment);
+                }
+
+                Thread.Sleep(1);
+            }
+        }
+
+        private TimeSpan? GetMomentDuration()
+        {
+            if (moment == null) return null;
+            if (next == null) return null;
+
+            return next.Utc - moment.Utc;
+        }
+
+        private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration)
+        {
+            var m = new ActivateMoment(moment.Utc, duration, momentCounter);
+
+            lock (handlersLock)
+            {
+                ActivateMomentHandlers(m);
+
+                foreach (var @event in moment.Events)
+                {
+                    ActivateEventHandlers(m, @event);
+                }
+            }
+
+            momentCounter++;
+        }
+
+        private void ActivateMomentHandlers(ActivateMoment m)
+        {
+            foreach (var handler in momentHandlers)
+            {
+                handler(m);
+            }
+        }
+
+        private void ActivateEventHandlers(ActivateMoment m, OverwatchEvent @event)
+        {
+            if (!eventHandlers.ContainsKey(@event.Type)) return;
+            var handlers = eventHandlers[@event.Type];
+
+            foreach (var handler in handlers)
+            {
+                handler(m, @event.Payload);
+            }
+        }
+
+        private OverwatchTranscript LoadModel(string inputFilename)
+        {
+            ZipFile.ExtractToDirectory(inputFilename, workingDir);
+
+            if (!File.Exists(transcriptFile))
+            {
+                closed = true;
+                throw new Exception("Is not a transcript file. Unzipped to: " + workingDir);
+            }
+
+            return JsonConvert.DeserializeObject<OverwatchTranscript>(File.ReadAllText(transcriptFile))!;
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been closed.");
+        }
+    }
+
+    public class ActivateMoment
+    {
+        public ActivateMoment(DateTime utc, TimeSpan? duration, long index)
+        {
+            Utc = utc;
+            Duration = duration;
+            Index = index;
+        }
+
+        public DateTime Utc { get; }
+        public TimeSpan? Duration { get; }
+        public long Index { get; }
+    }
+
+    public class ActivateEvent<T>
+    {
+        public ActivateEvent(ActivateMoment moment, T payload)
+        {
+            Moment = moment;
+            Payload = payload;
+        }
+
+        public ActivateMoment Moment { get; }
+        public T Payload { get; }
+    }
+}

Framework/OverwatchTranscript/TranscriptWriter.cs

@@ -0,0 +1,130 @@
+using Logging;
+using Newtonsoft.Json;
+using System.IO.Compression;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptWriter
+    {
+        void AddHeader(string key, object value);
+        void Add(DateTime utc, object payload);
+        void IncludeArtifact(string filePath);
+        void Write(string outputFilename);
+    }
+
+    public class TranscriptWriter : ITranscriptWriter
+    {
+        private readonly object _lock = new object();
+        private readonly MomentReferenceBuilder builder;
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly Dictionary<string, string> header = new Dictionary<string, string>();
+        private readonly BucketSet bucketSet;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private bool closed;
+
+        public TranscriptWriter(ILog log, string workingDir)
+        {
+            closed = false;
+            this.log = log;
+            this.workingDir = workingDir;
+            bucketSet = new BucketSet(log, workingDir);
+            builder = new MomentReferenceBuilder(log, workingDir);
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            CheckClosed();
+            bucketSet.Add(utc, payload);
+        }
+
+        public void AddHeader(string key, object value)
+        {
+            CheckClosed();
+            lock (_lock)
+            {
+                header.Add(key, Json.Serialize(value));
+            }
+        }
+
+        public void IncludeArtifact(string filePath)
+        {
+            CheckClosed();
+            if (!File.Exists(filePath)) throw new Exception("File not found: " + filePath);
+            if (!Directory.Exists(artifactsFolder)) Directory.CreateDirectory(artifactsFolder);
+            var name = Path.GetFileName(filePath);
+            File.Copy(filePath, Path.Combine(artifactsFolder, name), overwrite: false);
+        }
+
+        public void Write(string outputFilename)
+        {
+            CheckClosed();
+            closed = true;
+
+            var momentReferences = builder.Build(bucketSet.FinalizeBuckets());
+            var model = CreateModel(momentReferences);
+
+            File.WriteAllText(transcriptFile, Json.Serialize(model, Formatting.Indented));
+
+            ZipFile.CreateFromDirectory(workingDir, outputFilename);
+            log.Debug($"Transcript written to {outputFilename}");
+            log.Debug($"Common header: {Json.Serialize(model.Header.Common, Formatting.Indented)}");
+
+            Directory.Delete(workingDir, true);
+            log.Debug($"Workdir {workingDir} deleted");
+        }
+
+        private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences)
+        {
+            lock (_lock)
+            {
+                var model = new OverwatchTranscript
+                {
+                    Header = new OverwatchHeader
+                    {
+                        Common = CreateCommonHeader(momentReferences),
+                        Entries = header.Select(h =>
+                        {
+                            return new OverwatchHeaderEntry
+                            {
+                                Key = h.Key,
+                                Value = h.Value
+                            };
+                        }).ToArray()
+                    },
+                    MomentReferences = momentReferences
+                };
+
+                header.Clear();
+                return model;
+            }
+        }
+
+        private OverwatchCommonHeader CreateCommonHeader(OverwatchMomentReference[] momentReferences)
+        {
+            var moments = momentReferences.Sum(m => m.NumberOfMoments);
+            var events = momentReferences.Sum(m => m.NumberOfEvents);
+            var earliest = momentReferences.Min(m => m.EarliestUtc);
+            var latest = momentReferences.Max(m => m.LatestUtc);
+
+            return new OverwatchCommonHeader
+            {
+                NumberOfMoments = moments,
+                NumberOfEvents = events,
+                EarliestUtc = earliest,
+                LatestUtc = latest
+            };
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been written. Cannot modify or write again.");
+        }
+    }
+}

Framework/Utils/BlockInterval.cs

@@ -0,0 +1,31 @@
+namespace Utils
+{
+    public class BlockInterval
+    {
+        public BlockInterval(TimeRange timeRange, ulong from, ulong to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
+        }
+
+        public ulong From { get; }
+        public ulong To { get; }
+        public TimeRange TimeRange { get; }
+        public ulong NumberOfBlocks { get; }
+
+        public override string ToString()
+        {
+            return $"[{From} - {To}]";
+        }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -2,6 +2,9 @@
 {
     public class ByteSize
     {
+        public static readonly ByteSize Zero = new ByteSize(0);
+        public const double DefaultSecondsPerMB = 10.0;
+
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -10,8 +13,6 @@
 
         public long SizeInBytes { get; }
 
-        public const double DefaultSecondsPerMB = 10.0;
-
         public long ToMB()
         {
             return SizeInBytes / (1024 * 1024);

Framework/Utils/Formatter.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Globalization;
+
+namespace Utils
 {
     public static class Formatter
     {
@@ -10,7 +12,7 @@
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString() + sizeSuffixes[sizeOrder];
+            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,6 +2,7 @@
 {
     public class NumberSource
     {
+        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -11,8 +12,12 @@
 
         public int GetNextNumber()
         {
-            var n = number;
-            number++;
+            var n = -1;
+            lock (@lock)
+            {
+                n = number;
+                number++;
+            }
             return n;
         }
     }

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -3,13 +3,41 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
+        }
+
+        public static T[] Shuffled<T>(T[] items)
+        {
+            lock (@lock)
+            {
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
+            }
         }
     }
 }

Framework/Utils/Retry.cs

@@ -0,0 +1,135 @@
+namespace Utils
+{
+    public class Retry
+    {
+        private readonly string description;
+        private readonly TimeSpan maxTimeout;
+        private readonly TimeSpan sleepAfterFail;
+        private readonly Action<Failure> onFail;
+
+        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+        {
+            this.description = description;
+            this.maxTimeout = maxTimeout;
+            this.sleepAfterFail = sleepAfterFail;
+            this.onFail = onFail;
+        }
+
+        public void Run(Action task)
+        {
+            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+        }
+
+        public T Run<T>(Func<T> task)
+        {
+            T? result = default;
+
+            var run = new RetryRun(description, () =>
+            {
+                result = task();
+            }, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+
+            return result!;
+        }
+
+        private class RetryRun
+        {
+            private readonly string description;
+            private readonly Action task;
+            private readonly TimeSpan maxTimeout;
+            private readonly TimeSpan sleepAfterFail;
+            private readonly Action<Failure> onFail;
+            private readonly DateTime start = DateTime.UtcNow;
+            private readonly List<Failure> failures = new List<Failure>();
+            private int tryNumber;
+            private DateTime tryStart;
+
+            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+            {
+                this.description = description;
+                this.task = task;
+                this.maxTimeout = maxTimeout;
+                this.sleepAfterFail = sleepAfterFail;
+                this.onFail = onFail;
+
+                tryNumber = 0;
+                tryStart = DateTime.UtcNow;
+            }
+
+            public void Run()
+            {
+                while (true)
+                {
+                    CheckMaximums();
+
+                    tryNumber++;
+                    tryStart = DateTime.UtcNow;
+                    try
+                    {
+                        task();
+                        return;
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
+                    catch (Exception ex)
+                    {
+                        var failure = CaptureFailure(ex);
+                        onFail(failure);
+                        Time.Sleep(sleepAfterFail);
+                    }
+                }
+            }
+
+            private Failure CaptureFailure(Exception ex)
+            {
+                var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
+                failures.Add(f);
+                return f;
+            }
+
+            private void CheckMaximums()
+            {
+                if (Duration() > maxTimeout) Fail();
+            }
+
+            private void Fail()
+            {
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                        new AggregateException(failures.Select(f => f.Exception)));
+            }
+
+            private string GetFailureReport()
+            {
+                return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
+            }
+
+            private TimeSpan Duration()
+            {
+                return DateTime.UtcNow - start;
+            }
+        }
+    }
+
+    public class Failure
+    {
+        public Failure(Exception exception, TimeSpan duration, int tryNumber)
+        {
+            Exception = exception;
+            Duration = duration;
+            TryNumber = tryNumber;
+        }
+
+        public Exception Exception { get; }
+        public TimeSpan Duration { get; }
+        public int TryNumber { get; }
+
+        public string Describe()
+        {
+            return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
+        }
+    }
+}

Framework/Utils/RollingAverage.cs

@@ -0,0 +1,28 @@
+namespace Utils
+{
+    public static class RollingAverage
+    {
+        /// <param name="currentAverage">Value of average before new value is added.</param>
+        /// <param name="newNumberOfValues">Number of values in average after new value is added.</param>
+        /// <param name="newValue">New value to be added.</param>
+        /// <returns>New average value.</returns>
+        /// <exception cref="Exception">newNumberOfValues must be 1 or greater.</exception>
+        public static float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
+        {
+            if (newNumberOfValues < 1) throw new Exception("Should be at least 1 value.");
+
+            float n = newNumberOfValues;
+            var originalValue = currentAverage;
+            var originalValueWeight = ((n - 1.0f) / n);
+            var newValueWeight = (1.0f / n);
+            return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
+        }
+
+        public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
+        {
+            float totalWeight = weight1 + weight2;
+            if (totalWeight == 0.0f) return 0.0f;
+            return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
+        }
+    }
+}

Framework/Utils/Str.cs

@@ -0,0 +1,13 @@
+namespace Utils
+{
+    public static class Str
+    {
+        public static string Between(string input, string open, string close)
+        {
+            var openIndex = input.IndexOf(open) + open.Length;
+            var closeIndex = input.LastIndexOf(close);
+
+            return input.Substring(openIndex, closeIndex - openIndex);
+        }
+    }
+}

Framework/Utils/Time.cs

@@ -18,6 +18,12 @@
             task.Wait();
         }
 
+        public static string FormatDuration(TimeSpan? d)
+        {
+            if (d == null) return "[NULL]";
+            return FormatDuration(d.Value);
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -57,100 +63,70 @@
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate)
+        public static void WaitUntil(Func<bool> predicate, string msg)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
         {
             var start = DateTime.UtcNow;
+            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                if (DateTime.UtcNow - start > timeout)
+                var duration = DateTime.UtcNow - start;
+                if (duration > timeout)
                 {
-                    throw new TimeoutException("Operation timed out.");
+                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
+                tries++;
             }
         }
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, 1, description);
+            Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, 1, description);
+            return Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
-        public static void Retry(Action action, int maxRetries, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var retries = 0;
-            var exceptions = new List<Exception>();
-            while (true)
-            {
-                if (retries > maxRetries)
-                {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
-
-                try
-                {
-                    action();
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    retries++;
-                }
-
-                Sleep(retryTime);
-            }
+            Retry(action, maxTimeout, retryTime, description, f => { });
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var retries = 0;
-            var exceptions = new List<Exception>();
-            while (true)
-            {
-                if (retries > maxRetries)
-                {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
+            return Retry(action, maxTimeout, retryTime, description, f => { });
+        }
 
-                try
-                {
-                    return action();
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    retries++;
-                }
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            r.Run(action);
+        }
 
-                Sleep(retryTime);
-            }
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            return r.Run(action);
         }
     }
 }

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -0,0 +1,76 @@
+using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainEvents
+    {
+        private ChainEvents(
+            BlockInterval blockInterval,
+            Request[] requests,
+            RequestFulfilledEventDTO[] fulfilled,
+            RequestCancelledEventDTO[] cancelled,
+            RequestFailedEventDTO[] failed,
+            SlotFilledEventDTO[] slotFilled,
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
+            )
+        {
+            BlockInterval = blockInterval;
+            Requests = requests;
+            Fulfilled = fulfilled;
+            Cancelled = cancelled;
+            Failed = failed;
+            SlotFilled = slotFilled;
+            SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
+        }
+
+        public BlockInterval BlockInterval { get; }
+        public Request[] Requests { get; }
+        public RequestFulfilledEventDTO[] Fulfilled { get; }
+        public RequestCancelledEventDTO[] Cancelled { get; }
+        public RequestFailedEventDTO[] Failed { get; }
+        public SlotFilledEventDTO[] SlotFilled { get; }
+        public SlotFreedEventDTO[] SlotFreed { get; }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
+
+        public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
+        {
+            return FromContractEvents(contracts.GetEvents(blockInterval));
+        }
+
+        public static ChainEvents FromTimeRange(ICodexContracts contracts, TimeRange timeRange)
+        {
+            return FromContractEvents(contracts.GetEvents(timeRange));
+        }
+
+        public static ChainEvents FromContractEvents(ICodexContractsEvents events)
+        {
+            return new ChainEvents(
+                events.BlockInterval,
+                events.GetStorageRequests(),
+                events.GetRequestFulfilledEvents(),
+                events.GetRequestCancelledEvents(),
+                events.GetRequestFailedEvents(),
+                events.GetSlotFilledEvents(),
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
+            );
+        }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -0,0 +1,206 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+using NethereumWorkflow.BlockUtils;
+using System.Numerics;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateChangeHandler
+    {
+        void OnNewRequest(RequestEvent requestEvent);
+        void OnRequestFinished(RequestEvent requestEvent);
+        void OnRequestFulfilled(RequestEvent requestEvent);
+        void OnRequestCancelled(RequestEvent requestEvent);
+        void OnRequestFailed(RequestEvent requestEvent);
+        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
+        void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
+    }
+
+    public class RequestEvent
+    {
+        public RequestEvent(BlockTimeEntry block, IChainStateRequest request)
+        {
+            Block = block;
+            Request = request;
+        }
+
+        public BlockTimeEntry Block { get; }
+        public IChainStateRequest Request { get; }
+    }
+
+    public class ChainState
+    {
+        private readonly List<ChainStateRequest> requests = new List<ChainStateRequest>();
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly IChainStateChangeHandler handler;
+
+        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
+        {
+            this.log = new LogPrefixer(log, "(ChainState) ");
+            this.contracts = contracts;
+            handler = changeHandler;
+            TotalSpan = new TimeRange(startUtc, startUtc);
+        }
+
+        public TimeRange TotalSpan { get; private set; }
+        public IChainStateRequest[] Requests => requests.ToArray();
+
+        public int Update()
+        {
+            return Update(DateTime.UtcNow);
+        }
+
+        public int Update(DateTime toUtc)
+        {
+            var span = new TimeRange(TotalSpan.To, toUtc);
+            var events = ChainEvents.FromTimeRange(contracts, span);
+            Apply(events);
+
+            TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
+        }
+
+        private void Apply(ChainEvents events)
+        {
+            if (events.BlockInterval.TimeRange.From < TotalSpan.From)
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
+
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
+
+            // Run through each block and apply the events to the state in order.
+            var span = events.BlockInterval.TimeRange.Duration;
+            var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
+            var spanPerBlock = span / numBlocks;
+
+            var eventUtc = events.BlockInterval.TimeRange.From;
+            for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++)
+            {
+                var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
+                ApplyEvents(b, blockEvents, eventUtc);
+
+                eventUtc += spanPerBlock;
+            }
+        }
+
+        private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
+        {
+            foreach (var e in blockEvents)
+            {
+                dynamic d = e;
+                ApplyEvent(d);
+            }
+
+            ApplyTimeImplicitEvents(blockNumber, eventsUtc);
+        }
+
+        private void ApplyEvent(Request request)
+        {
+            if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
+                throw new Exception("Received NewRequest event for id that already exists.");
+
+            var newRequest = new ChainStateRequest(log, request, RequestState.New);
+            requests.Add(newRequest);
+
+            handler.OnNewRequest(new RequestEvent(request.Block, newRequest));
+        }
+
+        private void ApplyEvent(RequestFulfilledEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
+            handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
+        }
+
+        private void ApplyEvent(RequestCancelledEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
+            handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
+        }
+
+        private void ApplyEvent(RequestFailedEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
+            handler.OnRequestFailed(new RequestEvent(@event.Block, r));
+        }
+
+        private void ApplyEvent(SlotFilledEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotFreedEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Hosts.RemoveHost((int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
+        private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
+        {
+            foreach (var r in requests)
+            {
+                if (r.State == RequestState.Started
+                    && r.FinishedUtc < eventsUtc)
+                {
+                    r.UpdateState(blockNumber, RequestState.Finished);
+                    handler.OnRequestFinished(new RequestEvent(new BlockTimeEntry(blockNumber, eventsUtc), r));
+                }
+            }
+        }
+
+        private ChainStateRequest? FindRequest(IHasRequestId request)
+        {
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
+            return r;
+        }
+
+        private bool Equal(byte[] a, byte[] b)
+        {
+            return a.SequenceEqual(b);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -0,0 +1,60 @@
+using GethPlugin;
+using System.Numerics;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainStateChangeHandlerMux : IChainStateChangeHandler
+    {
+        public ChainStateChangeHandlerMux(params IChainStateChangeHandler[] handlers)
+        {
+            Handlers = handlers.ToList();
+        }
+
+        public List<IChainStateChangeHandler> Handlers { get; } = new List<IChainStateChangeHandler>();
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnNewRequest(requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestCancelled(requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFailed(requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFinished(requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFulfilled(requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFilled(requestEvent, host, slotIndex);
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -0,0 +1,80 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateRequest
+    {
+        Request Request { get; }
+        RequestState State { get; }
+        DateTime ExpiryUtc { get; }
+        DateTime FinishedUtc { get; }
+        EthAddress Client { get; }
+        RequestHosts Hosts { get; }
+    }
+
+    public class ChainStateRequest : IChainStateRequest
+    {
+        private readonly ILog log;
+
+        public ChainStateRequest(ILog log, Request request, RequestState state)
+        {
+            this.log = log;
+            Request = request;
+            State = state;
+
+            ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
+            FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
+
+            Log($"[{request.Block.BlockNumber}] Created as {State}.");
+
+            Client = new EthAddress(request.Client);
+            Hosts = new RequestHosts();
+        }
+
+        public Request Request { get; }
+        public RequestState State { get; private set; }
+        public DateTime ExpiryUtc { get; }
+        public DateTime FinishedUtc { get; }
+        public EthAddress Client { get; }
+        public RequestHosts Hosts { get; }
+
+        public void UpdateState(ulong blockNumber, RequestState newState)
+        {
+            Log($"[{blockNumber}] Transit: {State} -> {newState}");
+            State = newState;
+        }
+
+        public void Log(string msg)
+        {
+            log.Log($"Request '{Request.Id}': {msg}");
+        }
+    }
+
+    public class RequestHosts
+    {
+        private readonly Dictionary<int, EthAddress> hosts = new Dictionary<int, EthAddress>();
+
+        public void Add(EthAddress host, int index)
+        {
+            hosts.Add(index, host);
+        }
+        
+        public void RemoveHost(int index)
+        {
+            hosts.Remove(index);
+        }
+
+        public EthAddress? GetHost(int index)
+        {
+            if (!hosts.ContainsKey(index)) return null;
+            return hosts[index];
+        }
+
+        public EthAddress[] GetHosts()
+        {
+            return hosts.Values.ToArray();
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -0,0 +1,44 @@
+using GethPlugin;
+using System.Numerics;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class DoNothingChainEventHandler : IChainStateChangeHandler
+    {
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,9 +2,10 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
-using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -19,15 +20,13 @@ namespace CodexContractsPlugin
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
 
-        Request[] GetStorageRequests(TimeRange timeRange);
+        ICodexContractsEvents GetEvents(TimeRange timeRange);
+        ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
-        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
-        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
-        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
-        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
     }
 
+    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -63,7 +62,7 @@ namespace CodexContractsPlugin
 
         public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.TstWei, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -74,68 +73,17 @@ namespace CodexContractsPlugin
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
-            return balance.TestTokens();
+            return balance.TstWei();
         }
 
-        public Request[] GetStorageRequests(TimeRange timeRange)
+        public ICodexContractsEvents GetEvents(TimeRange timeRange)
         {
-            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            var i = StartInteraction();
-            return events
-                    .Select(e =>
-                    {
-                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+            return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange));
         }
 
-        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        public ICodexContractsEvents GetEvents(BlockInterval blockInterval)
         {
-            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
-        }
-
-        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
+            return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
         }
 
         public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
@@ -166,12 +114,6 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
-        private EthAddress GetEthAddressFromTransaction(string transactionHash)
-        {
-            var transaction = gethNode.GetTransaction(transactionHash);
-            return new EthAddress(transaction.From);
-        }
-
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -0,0 +1,113 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+using Nethereum.Contracts;
+using Nethereum.Hex.HexTypes;
+using NethereumWorkflow.BlockUtils;
+using Utils;
+
+namespace CodexContractsPlugin
+{
+    public interface ICodexContractsEvents
+    {
+        BlockInterval BlockInterval { get; }
+        Request[] GetStorageRequests();
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
+        RequestCancelledEventDTO[] GetRequestCancelledEvents();
+        RequestFailedEventDTO[] GetRequestFailedEvents();
+        SlotFilledEventDTO[] GetSlotFilledEvents();
+        SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
+    }
+
+    public class CodexContractsEvents : ICodexContractsEvents
+    {
+        private readonly ILog log;
+        private readonly IGethNode gethNode;
+        private readonly CodexContractsDeployment deployment;
+
+        public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval)
+        {
+            this.log = log;
+            this.gethNode = gethNode;
+            this.deployment = deployment;
+            BlockInterval = blockInterval;
+        }
+        
+        public BlockInterval BlockInterval { get; }
+
+        public Request[] GetStorageRequests()
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            var i = new ContractInteractions(log, gethNode);
+            return events.Select(e =>
+            {
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents()
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public RequestFailedEventDTO[] GetRequestFailedEvents()
+        {
+            var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents()
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents()
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
+        }
+
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            return gethNode.GetBlockForNumber(number);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,11 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Nethereum.Generators" Version="4.21.4" />
+    <PackageReference Include="Nethereum.Generators.Net" Version="4.21.4" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexContractsPlugin.Marketplace;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
@@ -24,7 +25,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,33 +60,46 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            });
+            }, nameof(DeployContract));
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
-            var abi = extractor.ExtractMarketplaceAbi();
+            var (abi, bytecode) = extractor.ExtractMarketplaceAbiAndByteCode();
+            EnsureCompatbility(abi, bytecode);
 
             var interaction = new ContractInteractions(tools.GetLog(), gethNode);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
             Log("Synced. Codex SmartContracts deployed.");
 
             return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
         }
 
+        private void EnsureCompatbility(string abi, string bytecode)
+        {
+            var expectedByteCode = MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant();
+
+            if (bytecode != expectedByteCode)
+            {
+                Log("Deployed contract is incompatible with current build of CodexContracts plugin. Running self-updater...");
+                var selfUpdater = new SelfUpdater();
+                selfUpdater.Update(abi, bytecode);
+            }
+        }
+
         private void Log(string msg)
         {
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -44,7 +44,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string MintTestTokens(EthAddress address, BigInteger amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             return MintTokens(address.Address, amount, tokenAddress);
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private string MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, BigInteger amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -93,7 +93,7 @@ namespace CodexContractsPlugin
             var function = new MintTokensFunction
             {
                 Holder = account,
-                Amount = amount.ToBig()
+                Amount = amount
             };
 
             return gethNode.SendTransaction(tokenAddress, function);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexContractsPlugin.Marketplace;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -30,14 +31,14 @@ namespace CodexContractsPlugin
             return marketplaceAddress;
         }
 
-        public string ExtractMarketplaceAbi()
+        public (string, string) ExtractMarketplaceAbiAndByteCode()
         {
             log.Debug();
-            var marketplaceAbi = Retry(FetchMarketplaceAbi);
-            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+            var (abi, bytecode) = Retry(FetchMarketplaceAbiAndByteCode);
+            if (string.IsNullOrEmpty(abi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
-            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
-            return marketplaceAbi;
+            log.Debug("Got Marketplace ABI: " + abi);
+            return (abi, bytecode);
         }
 
         private string FetchMarketplaceAddress()
@@ -47,16 +48,20 @@ namespace CodexContractsPlugin
             return marketplace!.address;
         }
 
-        private string FetchMarketplaceAbi()
+        private (string, string) FetchMarketplaceAbiAndByteCode()
         {
             var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            return abi!.ToString(Formatting.None);
+            var byteCode = artifact["bytecode"];
+            var abiResult = abi!.ToString(Formatting.None);
+            var byteCodeResult = byteCode!.ToString(Formatting.None).ToLowerInvariant().Replace("\"", "");
+          
+            return (abiResult, byteCodeResult);
         }
 
-        private static string Retry(Func<string> fetch)
+        private static T Retry<T>(Func<T> fetch)
         {
             return Time.Retry(fetch, nameof(ContractsContainerInfoExtractor));
         }

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,35 +1,73 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
 {
-    public partial class Request : RequestBase
+    public interface IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        BlockTimeEntry Block { get; set; }
+    }
+
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+
+        [JsonIgnore]
+        public string Id
+        {
+            get
+            {
+                return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant();
+            }
+        }
     }
 
-    public partial class RequestFulfilledEventDTO
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -1 +1,14 @@
 This code was generated using the Nethereum code generator, here: http://playground.nethereum.com
+
+1. Go to site -> Abi Code Gen.
+1. Contract name = "Marketplace".
+1. In container, get "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json".
+1. Save only ABI section as new JSON. (top-level is a json array.)
+1. From original JSON get byte code.
+1. Put ABI JSON and byte code into site.
+1. Generate.
+1. From site generated code, copy `public partial class MarketplaceDeployment` and everything after it. (be considerate of namespace brackets!)
+1. In Marketplace/Marketplace.cs, replace content of 'namespace CodexContractsPlugin.Marketplace'.
+
+
+

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -0,0 +1,96 @@
+using Utils;
+
+namespace CodexContractsPlugin
+{
+    public class SelfUpdater
+    {
+        public void Update(string abi, string bytecode)
+        {
+            var filePath = GetMarketplaceFilePath();
+            var content = GenerateContent(abi, bytecode);
+            var contentLines = content.Split("\r\n");
+
+            var beginWith = new string[]
+            {
+                "using Nethereum.ABI.FunctionEncoding.Attributes;",
+                "using Nethereum.Contracts;",
+                "using System.Numerics;",
+                "",
+                "// Generated code, do not modify.",
+                "",
+                "#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.",
+                "namespace CodexContractsPlugin.Marketplace",
+                "{"
+            };
+
+            var endWith = new string[]
+            {
+                "}",
+                "#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable."
+            };
+
+            File.Delete(filePath);
+            File.WriteAllLines(filePath,
+                beginWith.Concat(
+                contentLines.Concat(
+                endWith))
+            );
+
+            throw new Exception("Oh no! CodexContracts were updated. Current build of CodexContractsPlugin is incompatible. " +
+                "But fear not! SelfUpdater.cs has automatically updated the plugin. Just rebuild and rerun and it should work. " +
+                "Just in case, manual update instructions are found here: 'CodexContractsPlugin/Marketplace/README.md'.");
+        }
+
+        private string GetMarketplaceFilePath()
+        {
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
+        }
+
+        private string GenerateContent(string abi, string bytecode)
+        {
+            var deserializer = new Nethereum.Generators.Net.GeneratorModelABIDeserialiser();
+            var abiModel = deserializer.DeserialiseABI(abi);
+            var abiCtor = abiModel.Constructor;
+            var c = new Nethereum.Generators.CQS.ContractDeploymentCQSMessageGenerator(abiCtor, "namespace", bytecode, "Marketplace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+            var lines = "";
+            lines += c.GenerateClass();
+            lines += "\r\n";
+
+            foreach (var eventAbi in abiModel.Events)
+            {
+                var d = new Nethereum.Generators.DTOs.EventDTOGenerator(eventAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += d.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var errorAbi in abiModel.Errors)
+            {
+                var e = new Nethereum.Generators.DTOs.ErrorDTOGenerator(errorAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += e.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var funcAbi in abiModel.Functions)
+            {
+                var f = new Nethereum.Generators.DTOs.FunctionOutputDTOGenerator(funcAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                var ff = new Nethereum.Generators.CQS.FunctionCQSMessageGenerator(funcAbi, "namespace", "funcoutput", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += f.GenerateClass();
+                lines += "\r\n";
+                lines += ff.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var structAbi in abiModel.Structs)
+            {
+                var g = new Nethereum.Generators.DTOs.StructTypeGenerator(structAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += g.GenerateClass();
+                lines += "\r\n";
+            }
+
+            return lines;
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -1,45 +1,102 @@
-namespace CodexContractsPlugin
+using System.Numerics;
+
+namespace CodexContractsPlugin
 {
     public class TestToken : IComparable<TestToken>
     {
-        public TestToken(decimal amount)
+        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
+
+        public TestToken(BigInteger tstWei)
         {
-            Amount = amount;
+            TstWei = tstWei;
+            Tst = tstWei / WeiFactor;
         }
 
-        public decimal Amount { get; }
+        public BigInteger TstWei { get; }
+        public BigInteger Tst { get; }
 
         public int CompareTo(TestToken? other)
         {
-            return Amount.CompareTo(other!.Amount);
+            return TstWei.CompareTo(other!.TstWei);
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is TestToken token && Amount == token.Amount;
+            return obj is TestToken token && TstWei == token.TstWei;
         }
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(Amount);
+            return HashCode.Combine(TstWei);
         }
 
         public override string ToString()
         {
-            return $"{Amount} TestTokens";
+            var weiOnly = TstWei % WeiFactor;
+
+            var tokens = new List<string>();
+            if (Tst > 0) tokens.Add($"{Tst} TST");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
+
+            return string.Join(" + ", tokens);
+        }
+
+        public static TestToken operator +(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei + b.TstWei);
+        }
+
+        public static bool operator <(TestToken a, TestToken b)
+        {
+            return a.TstWei < b.TstWei;
+        }
+
+        public static bool operator >(TestToken a, TestToken b)
+        {
+            return a.TstWei > b.TstWei;
+        }
+
+        public static bool operator ==(TestToken a, TestToken b)
+        {
+            return a.TstWei == b.TstWei;
+        }
+
+        public static bool operator !=(TestToken a, TestToken b)
+        {
+            return a.TstWei != b.TstWei;
         }
     }
 
-    public static class TokensIntExtensions
+    public static class TestTokensExtensions
     {
-        public static TestToken TestTokens(this int i)
+        public static TestToken TstWei(this int i)
         {
-            return TestTokens(Convert.ToDecimal(i));
+            return TstWei(Convert.ToDecimal(i));
         }
 
-        public static TestToken TestTokens(this decimal i)
+        public static TestToken TstWei(this decimal i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this BigInteger i)
         {
             return new TestToken(i);
         }
+
+        public static TestToken Tst(this int i)
+        {
+            return Tst(Convert.ToDecimal(i));
+        }
+
+        public static TestToken Tst(this decimal i)
+        {
+            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this BigInteger i)
+        {
+            return new TestToken(i * TestToken.WeiFactor);
+        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,11 +1,13 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Utils;
 
 namespace CodexDiscordBotPlugin
 {
     public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
+        private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!";
         private readonly IPluginTools tools;
 
         public CodexDiscordBotPlugin(IPluginTools tools)
@@ -29,18 +31,76 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningContainers Deploy(DiscordBotStartupConfig config)
+        public RunningPod Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            return StartRewarderContainer(workflow, config);
+        }
+
+        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            WaitForStartupMessage(workflow, pod);
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
+        }
+
+        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
+            startupConfig.Add(config);
+            var pod = workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
+        }
+
+        private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod)
+        {
+            var finder = new LogLineFinder(ExpectedStartupMessage, workflow);
+            Time.WaitUntil(() =>
+            {
+                finder.FindLine(pod);
+                return finder.Found;
+            }, nameof(WaitForStartupMessage));
+        }
+
+        public class LogLineFinder : LogHandler
+        {
+            private readonly string message;
+            private readonly IStartupWorkflow workflow;
+
+            public LogLineFinder(string message, IStartupWorkflow workflow)
+            {
+                this.message = message;
+                this.workflow = workflow;
+            }
+
+            public void FindLine(RunningPod pod)
+            {
+                Found = false;
+                foreach (var c in pod.Containers)
+                {
+                    workflow.DownloadContainerLog(c, this);
+                    if (Found) return;
+                }
+            }
+
+            public bool Found { get; private set; }
+
+            protected override void ProcessLine(string line)
+            {
+                if (!Found && line.Contains(message)) Found = true;
+            }
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,11 +5,16 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
+        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        {
+            return Plugin(ci).DeployRewarder(config);
+        }
+
         private static CodexDiscordBotPlugin Plugin(CoreInterface ci)
         {
             return ci.GetPlugin<CodexDiscordBotPlugin>();