working as intended

# Conflicts:
#	cs-codex-dist-testing.sln

.github/workflows/docker-autoclient.yml

@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+

.github/workflows/docker-keymaker.yml

@@ -0,0 +1,27 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-KeyMaker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      docker_repo: codexstorage/codex-keymaker
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -104,7 +104,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: digests-${{ matrix.target.arch }}
-          path: /tmp/digests/*
+          path: /tmp/digests
           if-no-files-found: error
           retention-days: 1
 

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath

CONTRIBUTINGPLUGINS.MD

@@ -1,10 +1,26 @@
 # Distributed System Tests for Nim-Codex
 
 ## Contributing plugins
-The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
 1. Create a library assembly in the project plugins folder.
 1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
-1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
 
 ## Constructors & Tools
 Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
@@ -20,19 +36,34 @@ Your implementation of `IProjectPlugin` must have a public constructor with a si
    }
 ```
 
-`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
-
 ## Plugin Interfaces
 The `IProjectPlugin` interface requires the implementation of two methods.
-1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
-1. `Decommission` - Should your plugin have any active system resources, free them in this method.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
 
 There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
-1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
-1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
 
 ## Core Interface
-Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
 ```C#
    public static class CoreInterfaceExtensions
    {
@@ -48,12 +79,14 @@ Any functionality your plugin wants to expose to code which uses the framework w
    }
 ```
 
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
 While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
 
 ## Deploying, Wrapping, and Starting
 When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
-1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
-1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
 1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
 
 Here's an example:
@@ -69,8 +102,8 @@ public static class CoreInterfaceExtensions
 
       public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
       {
-         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
-         // represents the API of your project.
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
       }
 
       public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
@@ -82,5 +115,44 @@ public static class CoreInterfaceExtensions
    }
 ```
 
-The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
 

CONTRIBUTINGTESTS.MD

@@ -21,7 +21,7 @@ Do you want to write some tests using this distributed test setup? Great! Here's
    1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
 1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
 ### Continuous tests
-1. Add new code files to `Tests/CodexContinousTests/Tests`
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
 1. Inherrit from `ContinuousTest`
 1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
 1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)

Framework/ArgsUniform/ArgsUniform.cs

@@ -4,9 +4,8 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
+        private readonly Assigner<T> assigner;
         private readonly Action printAppInfo;
-        private readonly object? defaultsProvider;
-        private readonly IEnv.IEnv env;
         private readonly string[] args;
         private const int cliStart = 8;
         private const int shortStart = 38;
@@ -31,9 +30,9 @@ namespace ArgsUniform
         public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
             this.printAppInfo = printAppInfo;
-            this.defaultsProvider = defaultsProvider;
-            this.env = env;
             this.args = args;
+
+            assigner = new Assigner<T>(env, args, defaultsProvider);
         }
 
         public T Parse(bool printResult = false)
@@ -42,7 +41,7 @@ namespace ArgsUniform
             {
                 printAppInfo();
                 PrintHelp();
-                throw new Exception();
+                Environment.Exit(0);
             }
 
             var result = Activator.CreateInstance<T>();
@@ -53,18 +52,16 @@ namespace ArgsUniform
                 var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
                 if (attr != null)
                 {
-                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
                     {
-                        {
-                            missingRequired.Add(uniformProperty);
-                        }
+                        missingRequired.Add(uniformProperty);
                     }
                 }
             }
 
             if (missingRequired.Any())
             {
-                PrintResults(result, uniformProperties);
+                PrintResults(printResult,result, uniformProperties);
                 Print("");
                 foreach (var missing in missingRequired)
                 {
@@ -75,37 +72,39 @@ namespace ArgsUniform
                 }
 
                 PrintHelp();
-                throw new ArgumentException("Unable to assemble all required arguments");
+                Environment.Exit(1);
             }
 
-            if (printResult)
-            {
-                PrintResults(result, uniformProperties);
-            }
+            PrintResults(printResult, result, uniformProperties);
 
             return result;
         }
 
-        private void PrintResults(T result, PropertyInfo[] uniformProperties)
-        {
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
-            }
-            Print("");
-        }
-
         public void PrintHelp()
         {
             Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
-            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
-            foreach (var attr in attrs)
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
+            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            foreach (var prop in props)
             {
-                var a = attr!;
-                var optional = !a.Required ? " *" : "";
-                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+                var a = prop.GetCustomAttribute<UniformAttribute>();
+                if (a != null)
+                {
+                    var optional = !a.Required ? " (optional)" : "";
+                    var def = assigner.DescribeDefaultFor(prop);
+                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
+                }
+            }
+            Print("");
+        }
+
+        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
+        {
+            if (!printResult) return;
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
             }
             Print("");
         }
@@ -115,7 +114,7 @@ namespace ArgsUniform
             Console.WriteLine(msg);
         }
 
-        private void PrintAligned(string cli, string s, string env, string desc)
+        private void PrintAligned(string cli, string s, string env, string desc, string def)
         {
             Console.CursorLeft = cliStart;
             Console.Write(cli);
@@ -124,132 +123,8 @@ namespace ArgsUniform
             Console.CursorLeft = envStart;
             Console.Write(env);
             Console.CursorLeft = descStart;
-            Console.Write(desc + Environment.NewLine);
-        }
-
-        private object GetDefaultValue(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-
-        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var currentValue = uniformProperty.GetValue(result);
-            var isEmptryString = (currentValue as string) == string.Empty;
-            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
-
-            if (defaultsProvider == null) return false;
-
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return false;
-
-            var value = defaultProperty.GetValue(defaultsProvider);
-            if (value != null)
-            {
-                return Assign(result, uniformProperty, value);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-                    
-                    throw new NotSupportedException();
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
+            Console.Write(desc + " ");
+            Console.Write(def + Environment.NewLine);
         }
     }
 }

Framework/ArgsUniform/Assigner.cs

@@ -0,0 +1,186 @@
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class Assigner<T>
+    {
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private readonly object? defaultsProvider;
+
+        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
+        {
+            this.env = env;
+            this.args = args;
+            this.defaultsProvider = defaultsProvider;
+        }
+
+        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        public string DescribeDefaultFor(PropertyInfo property)
+        {
+            var obj = Activator.CreateInstance<T>();
+            var defaultValue = GetDefaultValue(obj, property);
+            if (defaultValue == null) return "";
+            if (defaultValue is string str)
+            {
+                return "\"" + str + "\"";
+            }
+            return defaultValue.ToString() ?? string.Empty;
+        }
+
+        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
+        {
+            // Get value from object's static initializer if it's there.
+            var currentValue = uniformProperty.GetValue(result);
+            if (currentValue != null) return currentValue;
+
+            // Get value from defaults-provider object if it's there.
+            if (defaultsProvider == null) return null;
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return null;
+            return defaultProperty.GetValue(defaultsProvider);
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var defaultValue = GetDefaultValue(result, uniformProperty);
+            var isEmptryString = (defaultValue as string) == string.Empty;
+            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
+            {
+                return Assign(result, uniformProperty, defaultValue);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
+
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+
+        private static object GetDefaultValueForType(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+    }
+}

Framework/Core/CoreInterface.cs

@@ -30,11 +30,11 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var file = entryPoint.Tools.GetLog().CreateSubfile();
-            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
-            var logHandler = new LogDownloadHandler(container.Name, file);
+            var msg = $"Downloading container log for '{container.Name}'";
+            entryPoint.Tools.GetLog().Log(msg);
+            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
             workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return logHandler.DownloadLog();
+            return new DownloadedLog(logHandler);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/DownloadedLog.cs

@@ -1,9 +1,11 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 
 namespace Core
 {
     public interface IDownloadedLog
     {
+        void IterateLines(Action<string> action);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
@@ -13,9 +15,22 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(LogFile logFile)
+        internal DownloadedLog(WriteToFileLogHandler logHandler)
         {
-            this.logFile = logFile;
+            logFile = logHandler.LogFile;
+        }
+
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
         }
 
         public string[] GetLinesContaining(string expectedString)

Framework/Core/Endpoint.cs

@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}

Framework/Core/EntryPoint.cs

@@ -38,10 +38,14 @@ namespace Core
             return new CoreInterface(this);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
         }
 
         internal T GetPlugin<T>() where T : IProjectPlugin

Framework/Core/Http.cs

@@ -1,22 +1,14 @@
 using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
 using Utils;
 
 namespace Core
 {
     public interface IHttp
     {
-        string HttpGetString(string route);
-        T HttpGetJson<T>(string route);
-        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
-        string HttpPostJson<TRequest>(string route, TRequest body);
-        TResponse HttpPostString<TResponse>(string route, string body);
-        string HttpPostStream(string route, Stream stream);
-        Stream HttpGetStream(string route);
-        T Deserialize<T>(string json);
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
     internal class Http : IHttp
@@ -24,192 +16,56 @@ namespace Core
         private static readonly object httpLock = new object();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
-        private readonly Address address;
-        private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
-        private readonly string? logAlias;
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        internal Http(ILog log, ITimeSet timeSet)
+            : this(log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.log = log;
             this.timeSet = timeSet;
-            this.address = address;
-            this.baseUrl = baseUrl;
             this.onClientCreated = onClientCreated;
-            this.logAlias = logAlias;
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
 
-        public string HttpGetString(string route)
+        public T OnClient<T>(Func<HttpClient, T> action)
         {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
+        {
+            var client = GetClient();
+
             return LockRetry(() =>
             {
-                return GetString(route);
-            }, $"HTTP-GET:{route}");
+                return action(client);
+            }, retry);
         }
 
-        public T HttpGetJson<T>(string route)
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
         {
-            return LockRetry(() =>
-            {
-                var json = GetString(route);
-                return Deserialize<T>(json);
-            }, $"HTTP-GET:{route}");
+            return new Endpoint(log, this, address, baseUrl, logAlias);
         }
 
-        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        private string GetDescription()
         {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                var json = Time.Wait(response.Content.ReadAsStringAsync());
-                if (!response.IsSuccessStatusCode)
-                {
-                    throw new HttpRequestException(json);
-                }
-                Log(GetUrl() + route, json);
-                return Deserialize<TResponse>(json);
-            }, $"HTTP-POST-JSON: {route}");
+            return DebugStack.GetCallerName(skipFrames: 2);
         }
 
-        public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public TResponse HttpPostString<TResponse>(string route, string body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJsonString(route, body);
-                if (response == null) throw new Exception("Received no response.");
-                var result = Deserialize<TResponse>(response);
-                if (result == null) throw new Exception("Failed to deserialize response");
-                return result;
-            }, $"HTTO-POST-JSON: {route}");
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return LockRetry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STREAM: {route}");
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return LockRetry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                return Time.Wait(client.GetStreamAsync(url));
-            }, $"HTTP-GET-STREAM: {route}");
-        }
-
-        public T Deserialize<T>(string json)
-        {
-            var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
-            {
-                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
-                {
-                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
-                    {
-                        errors.Add($"""
-                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
-                                    Path: {args.ErrorContext.Path}
-                                    Error: {args.ErrorContext.Error.Message}
-                                    """);
-                        args.ErrorContext.Handled = true;
-                    }
-                }
-            });
-            if (errors.Count > 0)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
-            }
-            else if (deserialized == null)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
-            }
-            return deserialized;
-        }
-
-        private string GetString(string route)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, "");
-            var result = Time.Wait(client.GetAsync(url));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            using var content = JsonContent.Create(body);
-            Log(url, JsonConvert.SerializeObject(body));
-            return Time.Wait(client.PostAsync(url, content));
-        }
-
-        private string PostJsonString(string route, string body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, body);
-            var content = new StringContent(body);
-            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-            var result = Time.Wait(client.PostAsync(url, content));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private string GetUrl()
-        {
-            return $"{address.Host}:{address.Port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            if (logAlias != null)
-            {
-                log.Debug($"({logAlias})({url}) = '{message}'", 3);
-            }
-            else
-            {
-                log.Debug($"({url}) = '{message}'", 3);
-            }
-        }
-
-        private T LockRetry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, Retry retry)
         {
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+                return retry.Run(operation);
             }
         }
 

Framework/Core/LogDownloadHandler.cs

@@ -1,28 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace Core
-{
-    internal class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly LogFile log;
-
-        internal LogDownloadHandler(string description, LogFile log)
-        {
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        internal IDownloadedLog DownloadLog()
-        {
-            return new DownloadedLog(log);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}

Framework/Core/PluginManager.cs

@@ -34,12 +34,12 @@
             return metadata;
         }
 
-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
             foreach (var pair in pairs)
             {
                 pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
             }
         }
 

Framework/Core/PluginTools.cs

@@ -1,13 +1,18 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
-using Utils;
 
 namespace Core
 {
     public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
     {
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+        ITimeSet TimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
     }
 
     public interface IWorkflowTool
@@ -22,9 +27,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp();
     }
 
     public interface IFileTool
@@ -34,37 +39,38 @@ namespace Core
 
     internal class PluginTools : IPluginTools
     {
-        private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
-        private ILog log;
+        private readonly LogPrefixer log;
 
         internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
         {
-            this.log = log;
+            this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            this.timeSet = timeSet;
+            TimeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
+        public ITimeSet TimeSet { get; }
+
         public void ApplyLogPrefix(string prefix)
         {
-            log = new LogPrefixer(log, prefix);
+            log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+            return CreateHttp(onClientCreated, TimeSet);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
+            return new Http(log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
+        public IHttp CreateHttp()
         {
-            return new Http(log, timeSet, address, baseUrl, logAlias);
+            return new Http(log, TimeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
@@ -72,9 +78,9 @@ namespace Core
             return workflowCreator.CreateWorkflow(namespaceOverride);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
             if (deleteTrackedFiles) fileManager.DeleteAllFiles();
         }
 

Framework/Core/TimeSet.cs

@@ -2,10 +2,31 @@
 {
     public interface ITimeSet
     {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
         TimeSpan HttpCallTimeout();
-        int HttpMaxNumberOfRetries();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
         TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
         TimeSpan K8sOperationTimeout();
     }
 
@@ -16,9 +37,9 @@
             return TimeSpan.FromMinutes(3);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 3;
+            return TimeSpan.FromMinutes(10);
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -26,7 +47,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -41,27 +62,27 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromHours(2);
+            return TimeSpan.FromMinutes(30);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 1;
+            return TimeSpan.FromHours(2.2);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(2);
+            return TimeSpan.FromSeconds(20);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(15);
+            return TimeSpan.FromHours(1);
         }
     }
 }

Framework/DiscordRewards/CheckConfig.cs

@@ -13,9 +13,9 @@ namespace DiscordRewards
     public enum CheckType
     {
         Uninitialized,
-        FilledSlot,
-        FinishedSlot,
-        PostedContract,
-        StartedContract,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,6 +3,13 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
+        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
+        }
     }
 
     public class RewardUsersCommand
@@ -10,4 +17,15 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class MarketAverage
+    {
+        public int NumberOfFinished { get; set; }
+        public int TimeRangeSeconds { get; set; }
+        public float Price { get; set; }
+        public float Size { get; set; }
+        public float Duration { get; set; }
+        public float Collateral { get; set; }
+        public float ProofProbability { get; set; }
+    }
 }

Framework/DiscordRewards/RewardRepo.cs

@@ -1,53 +1,53 @@
-using Utils;
-
-namespace DiscordRewards
+namespace DiscordRewards
 {
     public class RewardRepo
     {
         private static string Tag => RewardConfig.UsernameTag;
 
-        public RewardConfig[] Rewards { get; } = new RewardConfig[]
-        {
-            // Filled any slot
-            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
-            {
-                Type = CheckType.FilledSlot
-            }),
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];
 
-            // Finished any slot
-            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot
-            }),
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),
 
-            // Finished a sizable slot
-            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
-            }),
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),
 
-            // Posted any contract
-            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
-            {
-                Type = CheckType.PostedContract
-            }),
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),
 
-            // Started any contract
-            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.StartedContract
-            }),
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),
 
-            // Started a sizable contract
-            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot,
-                MinNumberOfHosts = 4,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
-            })
-        };
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
     }
 }

Framework/FileUtils/FileManager.cs

@@ -45,7 +45,7 @@ namespace FileUtils
         {
             var sw = Stopwatch.Begin(log);
             var result = GenerateRandomFile(size, label);
-            sw.End($"Generated file '{result.Describe()}'.");
+            sw.End($"Generated file {result.Describe()}.");
             return result;
         }
 
@@ -57,16 +57,27 @@ namespace FileUtils
         public void ScopedFiles(Action action)
         {
             PushFileSet();
-            action();
-            PopFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         public T ScopedFiles<T>(Func<T> action)
         {
             PushFileSet();
-            var result = action();
-            PopFileSet();
-            return result;
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         private void PushFileSet()

Framework/FileUtils/TrackedFile.cs

@@ -65,7 +65,7 @@ namespace FileUtils
 
                 if (readExpected == 0 && readActual == 0)
                 {
-                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
                     return;
                 }
 

Framework/KubernetesWorkflow/ByteSizeExtensions.cs

@@ -1,41 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public static class ByteSizeExtensions
-    {
-        public static string ToSuffixNotation(this ByteSize b)
-        {
-            long x = 1024;
-            var map = new Dictionary<long, string>
-            {
-                { Pow(x, 4), "Ti" },
-                { Pow(x, 3), "Gi" },
-                { Pow(x, 2), "Mi" },
-                { (x), "Ki" },
-            };
-
-            var bytes = b.SizeInBytes;
-            foreach (var pair in map)
-            {
-                if (bytes > pair.Key)
-                {
-                    double bytesD = bytes;
-                    double divD = pair.Key;
-                    double numD = Math.Ceiling(bytesD / divD);
-                    var v = Convert.ToInt64(numD);
-                    return $"{v}{pair.Value}";
-                }
-            }
-
-            return $"{bytes}";
-        }
-
-        private static long Pow(long x, int v)
-        {
-            long result = 1;
-            for (var i = 0; i < v; i++) result *= x;
-            return result;
-        }
-    }
-}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,7 +11,6 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
-        private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
@@ -27,11 +26,10 @@ namespace KubernetesWorkflow
             cts = new CancellationTokenSource();
         }
 
-        public void Start(ILogHandler logHandler)
+        public void Start()
         {
             if (worker != null) throw new InvalidOperationException();
 
-            this.logHandler = logHandler;
             cts = new CancellationTokenSource();
             worker = Task.Run(Worker);
         }
@@ -50,7 +48,9 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client);
+            var result = HasContainerBeenRestarted(client);
+            if (result) DownloadCrashedContainerLogs(client);
+            return result;
         }
 
         private void Worker()
@@ -83,14 +83,16 @@ namespace KubernetesWorkflow
         private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result) log.Log("Pod crash detected for " + containerName);
+            return result;
         }
 
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
-            log.Log("Pod crash detected for " + containerName);
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            logHandler!.Log(stream);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            handler.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         {
             var config = GetConfig();
             UpdateHostAddress(config);
+            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
             return config;
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,6 +43,11 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container);
+        }
+
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -59,14 +64,14 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
         {
             log.Debug();
 
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
             logHandler.Log(stream);
         }
 
@@ -110,7 +115,7 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteAllNamespacesStartingWith(string prefix)
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
         {
             log.Debug();
 
@@ -119,25 +124,28 @@ namespace KubernetesWorkflow
 
             foreach (var ns in namespaces)
             {
-                DeleteNamespace(ns);
+                DeleteNamespace(ns, wait);
             }
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(K8sNamespace))
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
             }
         }
 
-        public void DeleteNamespace(string ns)
+        public void DeleteNamespace(string ns, bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(ns))
             {
                 client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
             }
         }
 
@@ -372,7 +380,6 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -528,7 +535,7 @@ namespace KubernetesWorkflow
             }
             if (set.Memory.SizeInBytes != 0)
             {
-                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
             }
             return result;
         }
@@ -701,14 +708,14 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxRetries: 6,
+                maxTimeout: TimeSpan.FromMinutes(1),
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -864,16 +871,45 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
-        private void WaitUntilDeploymentOnline(string deploymentName)
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
+        private void WaitUntilDeploymentOnline(RunningContainer container)
         {
             WaitUntil(() =>
             {
-                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
+                CheckForCrash(container);
+
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            });
+            }, nameof(WaitUntilDeploymentOnline));
+        }
+
+        private void CheckForCrash(RunningContainer container)
+        {
+            var deploymentName = container.Recipe.Name;
+            var podName = GetPodName(container);
+
+            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
+            if (podInfo == null) return;
+            if (podInfo.Status == null) return;
+            if (podInfo.Status.ContainerStatuses == null) return;
+
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result)
+            {
+                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
+                log.Error(msg);
+
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+
+                throw new Exception(msg);
+            }
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -883,7 +919,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            });
+            }, nameof(WaitUntilDeploymentOffline));
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -892,19 +928,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            });
+            }, nameof(WaitUntilPodsForDeploymentAreOffline));
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
             }
             finally
             {
-                sw.End("", 1);
+                sw.End(msg, 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningContainers runningContainers);
-        void OnContainersStopped(RunningContainers runningContainers);
+        void OnContainersStarted(RunningPod runningPod);
+        void OnContainersStopped(RunningPod runningPod);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningContainers runningContainers)
+        public void OnContainersStarted(RunningPod runningPod)
         {
         }
 
-        public void OnContainersStopped(RunningContainers runningContainers)
+        public void OnContainersStopped(RunningPod runningPod)
         {
         }
 

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Logging;
+
+namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
@@ -20,4 +22,25 @@
 
         protected abstract void ProcessLine(string line);
     }
+
+    public class WriteToFileLogHandler : LogHandler, ILogHandler
+    {
+        public WriteToFileLogHandler(ILog sourceLog, string description)
+        {
+            LogFile = sourceLog.CreateSubfile();
+
+            var msg = $"{description} -->> {LogFile.FullFilename}";
+            sourceLog.Log(msg);
+
+            LogFile.Write(msg);
+            LogFile.WriteRaw(description);
+        }
+
+        public LogFile LogFile { get; }
+
+        protected override void ProcessLine(string line)
+        {
+            LogFile.WriteRaw(line);
+        }
+    }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -105,7 +105,7 @@ namespace KubernetesWorkflow.Recipe
 
         protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
         {
-            var size = 10.MB().ToSuffixNotation();
+            var size = 10.MB().SizeInBytes.ToString();
             volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
         }
 
@@ -114,7 +114,7 @@ namespace KubernetesWorkflow.Recipe
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                resourceQuantity: volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.SizeInBytes.ToString()));
         }
 
         protected void Additional(object userData)

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,8 +1,5 @@
-using k8s;
-using k8s.Models;
-using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Recipe;
 using KubernetesWorkflow.Types;
-using Newtonsoft.Json;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,16 +9,16 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningContainers containers);
+        PodInfo GetPodInfo(RunningPod pod);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers containers, bool waitTillStopped);
-        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
+        void Stop(RunningPod pod, bool waitTillStopped);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace();
-        void DeleteNamespacesStartingWith(string namespacePrefix);
+        void DeleteNamespace(bool wait);
+        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
     }
 
     public class StartupWorkflow : IStartupWorkflow
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,25 +60,36 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningContainers(startupConfig, startResult, containers);
+                var rc = new RunningPod(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return rc;
+                return new FutureContainers(rc, this);
+            });
+        }
+
+        public void WaitUntilOnline(RunningPod rc)
+        {
+            K8s(controller =>
+            {
+                foreach (var c in rc.Containers)
+                {
+                    controller.WaitUntilOnline(c);
+                }
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningContainers containers)
+        public PodInfo GetPodInfo(RunningPod pod)
         {
-            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -86,20 +97,20 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
+        public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
+                controller.Stop(runningPod.StartResult, waitTillStopped);
+                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
         }
 
@@ -111,19 +122,19 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteNamespace();
+                controller.DeleteNamespace(wait);
             });
         }
 
-        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow.Types
+{
+    public class FutureContainers
+    {
+        private readonly RunningPod runningPod;
+        private readonly StartupWorkflow workflow;
+
+        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
+        {
+            this.runningPod = runningPod;
+            this.workflow = workflow;
+        }
+
+        public RunningPod WaitForOnline()
+        {
+            workflow.WaitUntilOnline(runningPod);
+            return runningPod;
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
         public ContainerAddress[] Addresses { get; }
 
         [JsonIgnore]
-        public RunningContainers RunningContainers { get; internal set; } = null!;
+        public RunningPod RunningPod { get; internal set; } = null!;
 
         public Address GetAddress(ILog log, string portTag)
         {

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -2,15 +2,15 @@
 
 namespace KubernetesWorkflow.Types
 {
-    public class RunningContainers
+    public class RunningPod
     {
-        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
 
-            foreach (var c in containers) c.RunningContainers = this;
+            foreach (var c in containers) c.RunningPod = this;
         }
 
         public StartupConfig StartupConfig { get; }
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public string Name
         {
-            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
+            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
         }
 
         public string Describe()
@@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types
 
     public static class RunningContainersExtensions
     {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
+        public static string Describe(this RunningPod[] runningContainers)
         {
             return string.Join(",", runningContainers.Select(c => c.Describe()));
         }

Framework/Logging/LogPrefixer.cs

@@ -3,14 +3,21 @@
     public class LogPrefixer : ILog
     {
         private readonly ILog backingLog;
-        private readonly string prefix;
+
+        public LogPrefixer(ILog backingLog)
+        {
+            this.backingLog = backingLog;
+        }
 
         public LogPrefixer(ILog backingLog, string prefix)
         {
             this.backingLog = backingLog;
-            this.prefix = prefix;
+            Prefix = prefix;
         }
 
+        public string Prefix { get; set; } = string.Empty;
+
+
         public LogFile CreateSubfile(string ext = "log")
         {
             return backingLog.CreateSubfile(ext);
@@ -18,17 +25,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(prefix + message, skipFrames);
+            backingLog.Debug(Prefix + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(prefix + message);
+            backingLog.Error(Prefix + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(prefix + message);
+            backingLog.Log(Prefix + message);
         }
 
         public void AddStringReplace(string from, string to)

Framework/NethereumWorkflow/BlockUtils/BlockCache.cs

@@ -4,7 +4,7 @@
     {
         public delegate void CacheClearedEvent();
 
-        private const int MaxEntries = 1024;
+        private const int MaxEntries = 1024 * 1024 * 5;
         private readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
 
         public event CacheClearedEvent? OnCacheCleared;
@@ -35,5 +35,7 @@
             if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null;
             return value;
         }
+
+        public int Size { get { return entries.Count; } }
     }
 }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,13 +18,21 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
+        public BlockTimeEntry Get(ulong blockNumber)
+        {
+            bounds.Initialize();
+            var b = cache.Get(blockNumber);
+            if (b != null) return b;
+            return GetBlock(blockNumber);
+        }
+
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
             if (moment <= bounds.Genesis.Utc) return null;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector);
+            return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
         }
 
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
@@ -33,7 +41,16 @@ namespace NethereumWorkflow.BlockUtils
             if (moment >= bounds.Current.Utc) return null;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector);
+            return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;
+        }
+
+        private ulong Log(Func<ulong> operation)
+        {
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
+            var result = operation();
+            sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
+
+            return result;
         }
 
         private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func<DateTime, BlockTimeEntry, bool> isWhatIwant)
@@ -70,7 +87,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var next = GetBlock(entry.BlockNumber + 1);
             return
-                entry.Utc < target &&
+                entry.Utc <= target &&
                 next.Utc > target;
         }
 
@@ -78,7 +95,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var previous = GetBlock(entry.BlockNumber - 1);
             return
-                entry.Utc > target &&
+                entry.Utc >= target &&
                 previous.Utc < target;
         }
 

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -89,26 +89,9 @@ namespace NethereumWorkflow
             }
         }
 
-        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new()
         {
-            var wrapper = new Web3Wrapper(web3, log);
-            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
-
-            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
-            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
-
-            if (!fromBlock.HasValue)
-            {
-                log.Error("Failed to find lowest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-            if (!toBlock.HasValue)
-            {
-                log.Error("Failed to find highest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-
-            return GetEvents<TEvent>(address, fromBlock.Value, toBlock.Value);
+            return GetEvents<TEvent>(address, blockRange.From, blockRange.To);
         }
 
         public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
@@ -119,5 +102,32 @@ namespace NethereumWorkflow
             var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
             return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
         }
+
+        public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            if (fromBlock == null  || toBlock == null)
+            {
+                throw new Exception("Failed to convert time range to block range.");
+            }
+
+            return new BlockInterval(
+                timeRange: timeRange,
+                from: fromBlock.Value,
+                to: toBlock.Value
+            );
+        }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+            return blockTimeFinder.Get(number);
+        }
     }
 }

Framework/Utils/BlockInterval.cs

@@ -0,0 +1,30 @@
+namespace Utils
+{
+    public class BlockInterval
+    {
+        public BlockInterval(TimeRange timeRange, ulong from, ulong to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            TimeRange = timeRange;
+        }
+
+        public ulong From { get; }
+        public ulong To { get; }
+        public TimeRange TimeRange { get; }
+        public ulong NumberOfBlocks => To - From;
+
+        public override string ToString()
+        {
+            return $"[{From} - {To}]";
+        }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -2,6 +2,9 @@
 {
     public class ByteSize
     {
+        public static readonly ByteSize Zero = new ByteSize(0);
+        public const double DefaultSecondsPerMB = 10.0;
+
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -10,7 +13,6 @@
 
         public long SizeInBytes { get; }
 
-        public const double DefaultSecondsPerMB = 10.0;
 
         public long ToMB()
         {

Framework/Utils/Formatter.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Globalization;
+
+namespace Utils
 {
     public static class Formatter
     {
@@ -10,7 +12,7 @@
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString() + sizeSuffixes[sizeOrder];
+            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,6 +2,7 @@
 {
     public class NumberSource
     {
+        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -11,8 +12,12 @@
 
         public int GetNextNumber()
         {
-            var n = number;
-            number++;
+            var n = -1;
+            lock (@lock)
+            {
+                n = number;
+                number++;
+            }
             return n;
         }
     }

Framework/Utils/Retry.cs

@@ -0,0 +1,131 @@
+namespace Utils
+{
+    public class Retry
+    {
+        private readonly string description;
+        private readonly TimeSpan maxTimeout;
+        private readonly TimeSpan sleepAfterFail;
+        private readonly Action<Failure> onFail;
+
+        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+        {
+            this.description = description;
+            this.maxTimeout = maxTimeout;
+            this.sleepAfterFail = sleepAfterFail;
+            this.onFail = onFail;
+        }
+
+        public void Run(Action task)
+        {
+            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+        }
+
+        public T Run<T>(Func<T> task)
+        {
+            T? result = default;
+
+            var run = new RetryRun(description, () =>
+            {
+                result = task();
+            }, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+
+            return result!;
+        }
+
+        private class RetryRun
+        {
+            private readonly string description;
+            private readonly Action task;
+            private readonly TimeSpan maxTimeout;
+            private readonly TimeSpan sleepAfterFail;
+            private readonly Action<Failure> onFail;
+            private readonly DateTime start = DateTime.UtcNow;
+            private readonly List<Failure> failures = new List<Failure>();
+            private int tryNumber;
+            private DateTime tryStart;
+
+            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+            {
+                this.description = description;
+                this.task = task;
+                this.maxTimeout = maxTimeout;
+                this.sleepAfterFail = sleepAfterFail;
+                this.onFail = onFail;
+
+                tryNumber = 0;
+                tryStart = DateTime.UtcNow;
+            }
+
+            public void Run()
+            {
+                while (true)
+                {
+                    CheckMaximums();
+
+                    tryNumber++;
+                    tryStart = DateTime.UtcNow;
+                    try
+                    {
+                        task();
+                        return;
+                    }
+                    catch (Exception ex)
+                    {
+                        var failure = CaptureFailure(ex);
+                        onFail(failure);
+                        Time.Sleep(sleepAfterFail);
+                    }
+                }
+            }
+
+            private Failure CaptureFailure(Exception ex)
+            {
+                var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
+                failures.Add(f);
+                return f;
+            }
+
+            private void CheckMaximums()
+            {
+                if (Duration() > maxTimeout) Fail();
+            }
+
+            private void Fail()
+            {
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                        new AggregateException(failures.Select(f => f.Exception)));
+            }
+
+            private string GetFailureReport()
+            {
+                return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
+            }
+
+            private TimeSpan Duration()
+            {
+                return DateTime.UtcNow - start;
+            }
+        }
+    }
+
+    public class Failure
+    {
+        public Failure(Exception exception, TimeSpan duration, int tryNumber)
+        {
+            Exception = exception;
+            Duration = duration;
+            TryNumber = tryNumber;
+        }
+
+        public Exception Exception { get; }
+        public TimeSpan Duration { get; }
+        public int TryNumber { get; }
+
+        public string Describe()
+        {
+            return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
+        }
+    }
+}

Framework/Utils/Time.cs

@@ -18,6 +18,12 @@
             task.Wait();
         }
 
+        public static string FormatDuration(TimeSpan? d)
+        {
+            if (d == null) return "[NULL]";
+            return FormatDuration(d.Value);
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -57,100 +63,70 @@
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate)
+        public static void WaitUntil(Func<bool> predicate, string msg)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
         {
             var start = DateTime.UtcNow;
+            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                if (DateTime.UtcNow - start > timeout)
+                var duration = DateTime.UtcNow - start;
+                if (duration > timeout)
                 {
-                    throw new TimeoutException("Operation timed out.");
+                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
+                tries++;
             }
         }
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, 1, description);
+            Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, 1, description);
+            return Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
-        public static void Retry(Action action, int maxRetries, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var retries = 0;
-            var exceptions = new List<Exception>();
-            while (true)
-            {
-                if (retries > maxRetries)
-                {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
-
-                try
-                {
-                    action();
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    retries++;
-                }
-
-                Sleep(retryTime);
-            }
+            Retry(action, maxTimeout, retryTime, description, f => { });
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var retries = 0;
-            var exceptions = new List<Exception>();
-            while (true)
-            {
-                if (retries > maxRetries)
-                {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
+            return Retry(action, maxTimeout, retryTime, description, f => { });
+        }
 
-                try
-                {
-                    return action();
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    retries++;
-                }
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            r.Run(action);
+        }
 
-                Sleep(retryTime);
-            }
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            return r.Run(action);
         }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -0,0 +1,68 @@
+using CodexContractsPlugin.Marketplace;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainEvents
+    {
+        private ChainEvents(
+            BlockInterval blockInterval,
+            Request[] requests,
+            RequestFulfilledEventDTO[] fulfilled,
+            RequestCancelledEventDTO[] cancelled,
+            SlotFilledEventDTO[] slotFilled,
+            SlotFreedEventDTO[] slotFreed
+            )
+        {
+            BlockInterval = blockInterval;
+            Requests = requests;
+            Fulfilled = fulfilled;
+            Cancelled = cancelled;
+            SlotFilled = slotFilled;
+            SlotFreed = slotFreed;
+        }
+
+        public BlockInterval BlockInterval { get; }
+        public Request[] Requests { get; }
+        public RequestFulfilledEventDTO[] Fulfilled { get; }
+        public RequestCancelledEventDTO[] Cancelled { get; }
+        public SlotFilledEventDTO[] SlotFilled { get; }
+        public SlotFreedEventDTO[] SlotFreed { get; }
+
+        public IHasBlock[] All
+        {
+            get
+            {
+                var all = new List<IHasBlock>();
+                all.AddRange(Requests);
+                all.AddRange(Fulfilled);
+                all.AddRange(Cancelled);
+                all.AddRange(SlotFilled);
+                all.AddRange(SlotFreed);
+                return all.ToArray();
+            }
+        }
+
+        public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
+        {
+            return FromContractEvents(contracts.GetEvents(blockInterval));
+        }
+
+        public static ChainEvents FromTimeRange(ICodexContracts contracts, TimeRange timeRange)
+        {
+            return FromContractEvents(contracts.GetEvents(timeRange));
+        }
+
+        public static ChainEvents FromContractEvents(ICodexContractsEvents events)
+        {
+            return new ChainEvents(
+                events.BlockInterval,
+                events.GetStorageRequests(),
+                events.GetRequestFulfilledEvents(),
+                events.GetRequestCancelledEvents(),
+                events.GetSlotFilledEvents(),
+                events.GetSlotFreedEvents()
+            );
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -0,0 +1,170 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+using NethereumWorkflow.BlockUtils;
+using System.Numerics;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateChangeHandler
+    {
+        void OnNewRequest(RequestEvent requestEvent);
+        void OnRequestFinished(RequestEvent requestEvent);
+        void OnRequestFulfilled(RequestEvent requestEvent);
+        void OnRequestCancelled(RequestEvent requestEvent);
+        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
+        void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+    }
+
+    public class RequestEvent
+    {
+        public RequestEvent(BlockTimeEntry block, IChainStateRequest request)
+        {
+            Block = block;
+            Request = request;
+        }
+
+        public BlockTimeEntry Block { get; }
+        public IChainStateRequest Request { get; }
+    }
+
+    public class ChainState
+    {
+        private readonly List<ChainStateRequest> requests = new List<ChainStateRequest>();
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly IChainStateChangeHandler handler;
+
+        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
+        {
+            this.log = new LogPrefixer(log, "(ChainState) ");
+            this.contracts = contracts;
+            handler = changeHandler;
+            StartUtc = startUtc;
+            TotalSpan = new TimeRange(startUtc, startUtc);
+        }
+
+        public TimeRange TotalSpan { get; private set; }
+        public IChainStateRequest[] Requests => requests.ToArray();
+
+        public DateTime StartUtc { get; }
+
+        public void Update()
+        {
+            Update(DateTime.UtcNow);
+        }
+
+        public void Update(DateTime toUtc)
+        {
+            var span = new TimeRange(TotalSpan.To, toUtc);
+            var events = ChainEvents.FromTimeRange(contracts, span);
+            Apply(events);
+
+            TotalSpan = new TimeRange(TotalSpan.From, span.To);
+        }
+
+        private void Apply(ChainEvents events)
+        {
+            if (events.BlockInterval.TimeRange.From < TotalSpan.From)
+                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+
+            log.Log($"ChainState updating: {events.BlockInterval}");
+
+            // Run through each block and apply the events to the state in order.
+            var span = events.BlockInterval.TimeRange.Duration;
+            var numBlocks = events.BlockInterval.NumberOfBlocks;
+            var spanPerBlock = span / numBlocks;
+
+            var eventUtc = events.BlockInterval.TimeRange.From;
+            for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++)
+            {
+                var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
+                ApplyEvents(b, blockEvents, eventUtc);
+
+                eventUtc += spanPerBlock;
+            }
+        }
+
+        private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
+        {
+            foreach (var e in blockEvents)
+            {
+                dynamic d = e;
+                ApplyEvent(d);
+            }
+
+            ApplyTimeImplicitEvents(blockNumber, eventsUtc);
+        }
+
+        private void ApplyEvent(Request request)
+        {
+            if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
+                throw new Exception("Received NewRequest event for id that already exists.");
+
+            var newRequest = new ChainStateRequest(log, request, RequestState.New);
+            requests.Add(newRequest);
+
+            handler.OnNewRequest(new RequestEvent(request.Block, newRequest));
+        }
+
+        private void ApplyEvent(RequestFulfilledEventDTO @event)
+        {
+            var r = FindRequest(@event.RequestId);
+            if (r == null) return;
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
+            handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
+        }
+
+        private void ApplyEvent(RequestCancelledEventDTO @event)
+        {
+            var r = FindRequest(@event.RequestId);
+            if (r == null) return;
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
+            handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
+        }
+
+        private void ApplyEvent(SlotFilledEventDTO @event)
+        {
+            var r = FindRequest(@event.RequestId);
+            if (r == null) return;
+            r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotFreedEventDTO @event)
+        {
+            var r = FindRequest(@event.RequestId);
+            if (r == null) return;
+            r.Hosts.RemoveHost((int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
+        private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
+        {
+            foreach (var r in requests)
+            {
+                if (r.State == RequestState.Started
+                    && r.FinishedUtc < eventsUtc)
+                {
+                    r.UpdateState(blockNumber, RequestState.Finished);
+                    handler.OnRequestFinished(new RequestEvent(new BlockTimeEntry(blockNumber, eventsUtc), r));
+                }
+            }
+        }
+
+        private ChainStateRequest? FindRequest(byte[] requestId)
+        {
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
+            if (r == null) log.Log("Unable to find request by ID!");
+            return r;
+        }
+
+        private bool Equal(byte[] a, byte[] b)
+        {
+            return a.SequenceEqual(b);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -0,0 +1,80 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateRequest
+    {
+        Request Request { get; }
+        RequestState State { get; }
+        DateTime ExpiryUtc { get; }
+        DateTime FinishedUtc { get; }
+        EthAddress Client { get; }
+        RequestHosts Hosts { get; }
+    }
+
+    public class ChainStateRequest : IChainStateRequest
+    {
+        private readonly ILog log;
+
+        public ChainStateRequest(ILog log, Request request, RequestState state)
+        {
+            this.log = log;
+            Request = request;
+            State = state;
+
+            ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
+            FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
+
+            Log($"[{request.Block.BlockNumber}] Created as {State}.");
+
+            Client = new EthAddress(request.Client);
+            Hosts = new RequestHosts();
+        }
+
+        public Request Request { get; }
+        public RequestState State { get; private set; }
+        public DateTime ExpiryUtc { get; }
+        public DateTime FinishedUtc { get; }
+        public EthAddress Client { get; }
+        public RequestHosts Hosts { get; }
+
+        public void UpdateState(ulong blockNumber, RequestState newState)
+        {
+            Log($"[{blockNumber}] Transit: {State} -> {newState}");
+            State = newState;
+        }
+
+        public void Log(string msg)
+        {
+            log.Log($"Request '{Request.Id}': {msg}");
+        }
+    }
+
+    public class RequestHosts
+    {
+        private readonly Dictionary<int, EthAddress> hosts = new Dictionary<int, EthAddress>();
+
+        public void Add(EthAddress host, int index)
+        {
+            hosts.Add(index, host);
+        }
+        
+        public void RemoveHost(int index)
+        {
+            hosts.Remove(index);
+        }
+
+        public EthAddress? GetHost(int index)
+        {
+            if (!hosts.ContainsKey(index)) return null;
+            return hosts[index];
+        }
+
+        public EthAddress[] GetHosts()
+        {
+            return hosts.Values.ToArray();
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -0,0 +1,32 @@
+using GethPlugin;
+using System.Numerics;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class DoNothingChainEventHandler : IChainStateChangeHandler
+    {
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,9 +2,10 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
-using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -19,15 +20,13 @@ namespace CodexContractsPlugin
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
 
-        Request[] GetStorageRequests(TimeRange timeRange);
+        ICodexContractsEvents GetEvents(TimeRange timeRange);
+        ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
-        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
-        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
-        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
-        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
     }
 
+    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -63,7 +62,7 @@ namespace CodexContractsPlugin
 
         public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.TstWei, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -74,68 +73,17 @@ namespace CodexContractsPlugin
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
-            return balance.TestTokens();
+            return balance.TstWei();
         }
 
-        public Request[] GetStorageRequests(TimeRange timeRange)
+        public ICodexContractsEvents GetEvents(TimeRange timeRange)
         {
-            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            var i = StartInteraction();
-            return events
-                    .Select(e =>
-                    {
-                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+            return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange));
         }
 
-        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        public ICodexContractsEvents GetEvents(BlockInterval blockInterval)
         {
-            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
-        }
-
-        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
-        {
-            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
-                return result;
-            }).ToArray();
+            return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
         }
 
         public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
@@ -166,12 +114,6 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
-        private EthAddress GetEthAddressFromTransaction(string transactionHash)
-        {
-            var transaction = gethNode.GetTransaction(transactionHash);
-            return new EthAddress(transaction.From);
-        }
-
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -0,0 +1,108 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+using Nethereum.Hex.HexTypes;
+using NethereumWorkflow.BlockUtils;
+using Utils;
+
+namespace CodexContractsPlugin
+{
+    public interface ICodexContractsEvents
+    {
+        BlockInterval BlockInterval { get; }
+        Request[] GetStorageRequests();
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
+        RequestCancelledEventDTO[] GetRequestCancelledEvents();
+        SlotFilledEventDTO[] GetSlotFilledEvents();
+        SlotFreedEventDTO[] GetSlotFreedEvents();
+    }
+
+    public class CodexContractsEvents : ICodexContractsEvents
+    {
+        private readonly ILog log;
+        private readonly IGethNode gethNode;
+        private readonly CodexContractsDeployment deployment;
+
+        public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval)
+        {
+            this.log = log;
+            this.gethNode = gethNode;
+            this.deployment = deployment;
+            BlockInterval = blockInterval;
+        }
+        
+        public BlockInterval BlockInterval { get; }
+
+        public Request[] GetStorageRequests()
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            var i = new ContractInteractions(log, gethNode);
+            return events
+            .Select(e =>
+            {
+                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents()
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents()
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents()
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            return gethNode.GetBlockForNumber(number);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,7 +59,7 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            });
+            }, nameof(DeployContract));
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@@ -71,7 +71,7 @@ namespace CodexContractsPlugin
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
             Log("Synced. Codex SmartContracts deployed.");
 
@@ -83,9 +83,9 @@ namespace CodexContractsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -44,7 +44,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string MintTestTokens(EthAddress address, BigInteger amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             return MintTokens(address.Address, amount, tokenAddress);
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private string MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, BigInteger amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -93,7 +93,7 @@ namespace CodexContractsPlugin
             var function = new MintTokensFunction
             {
                 Holder = account,
-                Amount = amount.ToBig()
+                Amount = amount
             };
 
             return gethNode.SendTransaction(tokenAddress, function);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexContractsPlugin.Marketplace;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -53,7 +54,18 @@ namespace CodexContractsPlugin
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            return abi!.ToString(Formatting.None);
+            var byteCode = artifact["bytecode"];
+            var abiResult = abi!.ToString(Formatting.None);
+            var byteCodeResult = byteCode!.ToString(Formatting.None);
+
+            if (byteCodeResult
+                .ToLowerInvariant()
+                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
+            {
+                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
+            }
+
+            return abiResult;
         }
 
         private static string Retry(Func<string> fetch)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,35 +1,56 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
 {
-    public partial class Request : RequestBase
+    public interface IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+
+        [JsonIgnore]
+        public string Id
+        {
+            get
+            {
+                return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant();
+            }
+        }
     }
 
-    public partial class RequestFulfilledEventDTO
+    public partial class RequestFulfilledEventDTO : IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO
+    public partial class RequestCancelledEventDTO : IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO
+    public partial class SlotFilledEventDTO : IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO
+    public partial class SlotFreedEventDTO : IHasBlock
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -1 +1,14 @@
 This code was generated using the Nethereum code generator, here: http://playground.nethereum.com
+
+1. Go to site -> Abi Code Gen.
+1. Contract name = "Marketplace".
+1. In container, get "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json".
+1. Save only ABI section as new JSON. (top-level is a json array.)
+1. From original JSON get byte code.
+1. Put ABI JSON and byte code into site.
+1. Generate.
+1. From site generated code, copy `public partial class MarketplaceDeployment` and everything after it. (be considerate of namespace brackets!)
+1. In Marketplace/Marketplace.cs, replace content of 'namespace CodexContractsPlugin.Marketplace'.
+
+
+

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -1,45 +1,102 @@
-namespace CodexContractsPlugin
+using System.Numerics;
+
+namespace CodexContractsPlugin
 {
     public class TestToken : IComparable<TestToken>
     {
-        public TestToken(decimal amount)
+        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
+
+        public TestToken(BigInteger tstWei)
         {
-            Amount = amount;
+            TstWei = tstWei;
+            Tst = tstWei / WeiFactor;
         }
 
-        public decimal Amount { get; }
+        public BigInteger TstWei { get; }
+        public BigInteger Tst { get; }
 
         public int CompareTo(TestToken? other)
         {
-            return Amount.CompareTo(other!.Amount);
+            return TstWei.CompareTo(other!.TstWei);
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is TestToken token && Amount == token.Amount;
+            return obj is TestToken token && TstWei == token.TstWei;
         }
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(Amount);
+            return HashCode.Combine(TstWei);
         }
 
         public override string ToString()
         {
-            return $"{Amount} TestTokens";
+            var weiOnly = TstWei % WeiFactor;
+
+            var tokens = new List<string>();
+            if (Tst > 0) tokens.Add($"{Tst} TST");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
+
+            return string.Join(" + ", tokens);
+        }
+
+        public static TestToken operator +(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei + b.TstWei);
+        }
+
+        public static bool operator <(TestToken a, TestToken b)
+        {
+            return a.TstWei < b.TstWei;
+        }
+
+        public static bool operator >(TestToken a, TestToken b)
+        {
+            return a.TstWei > b.TstWei;
+        }
+
+        public static bool operator ==(TestToken a, TestToken b)
+        {
+            return a.TstWei == b.TstWei;
+        }
+
+        public static bool operator !=(TestToken a, TestToken b)
+        {
+            return a.TstWei != b.TstWei;
         }
     }
 
-    public static class TokensIntExtensions
+    public static class TestTokensExtensions
     {
-        public static TestToken TestTokens(this int i)
+        public static TestToken TstWei(this int i)
         {
-            return TestTokens(Convert.ToDecimal(i));
+            return TstWei(Convert.ToDecimal(i));
         }
 
-        public static TestToken TestTokens(this decimal i)
+        public static TestToken TstWei(this decimal i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this BigInteger i)
         {
             return new TestToken(i);
         }
+
+        public static TestToken Tst(this int i)
+        {
+            return Tst(Convert.ToDecimal(i));
+        }
+
+        public static TestToken Tst(this decimal i)
+        {
+            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this BigInteger i)
+        {
+            return new TestToken(i * TestToken.WeiFactor);
+        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,11 +1,13 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Utils;
 
 namespace CodexDiscordBotPlugin
 {
     public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
+        private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!";
         private readonly IPluginTools tools;
 
         public CodexDiscordBotPlugin(IPluginTools tools)
@@ -29,18 +31,76 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningContainers Deploy(DiscordBotStartupConfig config)
+        public RunningPod Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            return StartRewarderContainer(workflow, config);
+        }
+
+        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            WaitForStartupMessage(workflow, pod);
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
+        }
+
+        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
+            startupConfig.Add(config);
+            var pod = workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
+        }
+
+        private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod)
+        {
+            var finder = new LogLineFinder(ExpectedStartupMessage, workflow);
+            Time.WaitUntil(() =>
+            {
+                finder.FindLine(pod);
+                return finder.Found;
+            }, nameof(WaitForStartupMessage));
+        }
+
+        public class LogLineFinder : LogHandler
+        {
+            private readonly string message;
+            private readonly IStartupWorkflow workflow;
+
+            public LogLineFinder(string message, IStartupWorkflow workflow)
+            {
+                this.message = message;
+                this.workflow = workflow;
+            }
+
+            public void FindLine(RunningPod pod)
+            {
+                Found = false;
+                foreach (var c in pod.Containers)
+                {
+                    workflow.DownloadContainerLog(c, this);
+                    if (Found) return;
+                }
+            }
+
+            public bool Found { get; private set; }
+
+            protected override void ProcessLine(string line)
+            {
+                if (!Found && line.Contains(message)) Found = true;
+            }
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,11 +5,16 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
+        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        {
+            return Plugin(ci).DeployRewarder(config);
+        }
+
         private static CodexDiscordBotPlugin Plugin(CoreInterface ci)
         {
             return ci.GetPlugin<CodexDiscordBotPlugin>();

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,9 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "thatbenbierens/codex-discordbot:initial";
+        public override string Image => "codexstorage/codex-discordbot:sha-8033da1";
+
+        public static string RewardsPort = "bot_rewards_port";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -19,6 +21,7 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);
             AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("REWARDSCHANNELNAME", config.RewardChannelName);
             AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
             AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
 
@@ -30,13 +33,15 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
             AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
+            AddEnvVar("NODISCORD", "1");
+
+            AddInternalPortAndVar("REWARDAPIPORT", RewardsPort);
+
             if (!string.IsNullOrEmpty(config.DataPath))
             {
                 AddEnvVar("DATAPATH", config.DataPath);
                 AddVolume(config.DataPath, 1.GB());
             }
-
-            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -2,7 +2,7 @@
 {
     public class DiscordBotStartupConfig
     {
-        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo, string rewardChannelName)
         {
             Name = name;
             Token = token;
@@ -11,6 +11,7 @@
             AdminChannelName = adminChannelName;
             KubeNamespace = kubeNamespace;
             GethInfo = gethInfo;
+            RewardChannelName = rewardChannelName;
         }
 
         public string Name { get; }
@@ -18,11 +19,34 @@
         public string ServerName { get; }
         public string AdminRoleName { get; }
         public string AdminChannelName { get; }
+        public string RewardChannelName { get; }
         public string KubeNamespace { get; }
         public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }
     }
 
+    public class RewarderBotStartupConfig
+    {
+        public RewarderBotStartupConfig(string name, string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
+        {
+            Name = name;
+            DiscordBotHost = discordBotHost;
+            DiscordBotPort = discordBotPort;
+            IntervalMinutes = intervalMinutes;
+            HistoryStartUtc = historyStartUtc;
+            GethInfo = gethInfo;
+            DataPath = dataPath;
+        }
+
+        public string Name { get; }
+        public string DiscordBotHost { get; }
+        public int DiscordBotPort { get; }
+        public int IntervalMinutes { get; }
+        public DateTime HistoryStartUtc { get; }
+        public DiscordBotGethInfo GethInfo { get; }
+        public string? DataPath { get; set; }
+    }
+
     public class DiscordBotGethInfo
     {
         public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -0,0 +1,39 @@
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow;
+using Utils;
+
+namespace CodexDiscordBotPlugin
+{
+    public class RewarderBotContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "discordbot-rewarder";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<RewarderBotStartupConfig>();
+
+            SetSchedulingAffinity(notIn: "false");
+
+            AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost);
+            AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString());
+            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes.ToString());
+            var offset = new DateTimeOffset(config.HistoryStartUtc);
+            AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString());
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
+
+            if (!string.IsNullOrEmpty(config.DataPath))
+            {
+                AddEnvVar("DATAPATH", config.DataPath);
+                AddVolume(config.DataPath, 1.GB());
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -0,0 +1,92 @@
+using Core;
+using KubernetesWorkflow.Types;
+using Logging;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace CodexPlugin
+{
+    public class ApiChecker
+    {
+        // <INSERT-OPENAPI-YAML-HASH>
+        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiFilePath = "/codex/openapi.yaml";
+        private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
+
+        private const bool Disable = false;
+
+        private const string Warning =
+            "Warning: CodexPlugin was unable to find the openapi.yaml file in the Codex container. Are you running an old version of Codex? " +
+            "Plugin will continue as normal, but API compatibility is not guaranteed!";
+
+        private const string Failure =
+            "Codex API compatibility check failed! " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
+
+        private static bool checkPassed = false;
+
+        private readonly IPluginTools pluginTools;
+        private readonly ILog log;
+
+        public ApiChecker(IPluginTools pluginTools)
+        {
+            this.pluginTools = pluginTools;
+            log = pluginTools.GetLog();
+
+            if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
+        }
+
+        public void CheckCompatibility(RunningPod[] containers)
+        {
+            if (checkPassed) return;
+
+            Log("CodexPlugin is checking API compatibility...");
+
+            if (Disable || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(DisableEnvironmentVariable)))
+            {
+                Log("API compatibility checking has been disabled.");
+                checkPassed = true;
+                return;
+            }
+
+            var workflow = pluginTools.CreateWorkflow();
+            var container = containers.First().Containers.First();
+            var containerApi = workflow.ExecuteCommand(container, "cat", OpenApiFilePath);
+
+            if (string.IsNullOrEmpty(containerApi))
+            {
+                log.Error(Warning);
+
+                checkPassed = true;
+                return;
+            }
+
+            var containerHash = Hash(containerApi);
+            if (containerHash == OpenApiYamlHash)
+            {
+                Log("API compatibility check passed.");
+                checkPassed = true;
+                return;
+            }
+
+            log.Error(Failure);
+            throw new Exception(Failure);
+        }
+
+        private string Hash(string file)
+        {
+            var fileBytes = Encoding.ASCII.GetBytes(file
+                .Replace(Environment.NewLine, ""));
+            var sha = SHA256.Create();
+            var hash = sha.ComputeHash(fileBytes);
+            return BitConverter.ToString(hash);
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,99 +1,124 @@
-using Core;
+using CodexOpenApi;
+using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
 {
-    public class CodexAccess : ILogHandler
+    public class CodexAccess
     {
+        private readonly ILog log;
         private readonly IPluginTools tools;
-        private bool hasContainerCrashed;
+        private readonly Mapper mapper = new Mapper();
 
-        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
+        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
+            log = tools.GetLog();
             Container = container;
             CrashWatcher = crashWatcher;
-            hasContainerCrashed = false;
 
-            CrashWatcher.Start(this);
+            CrashWatcher.Start();
         }
 
-        public RunningContainer Container { get; }
+        public RunningPod Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
-            var http = Http();
-            var str = http.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
+            // Cannot use openAPI: debug/peer endpoint is not specified there.
+            return CrashCheck(() =>
             {
-                return new CodexDebugPeerResponse
+                var endpoint = GetEndpoint();
+                var str = endpoint.HttpGetString($"debug/peer/{peerId}");
+
+                if (str.ToLowerInvariant() == "unable to find peer!")
                 {
-                    IsPeerFound = false
-                };
-            }
+                    return new DebugPeer
+                    {
+                        IsPeerFound = false
+                    };
+                }
 
-            var result = http.Deserialize<CodexDebugPeerResponse>(str);
-            result.IsPeerFound = true;
-            return result;
+                var result = endpoint.Deserialize<DebugPeer>(str);
+                result.IsPeerFound = true;
+                return result;
+            });
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
         {
-            return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
+            OnCodex(api =>
+            {
+                Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
+                return Task.FromResult(string.Empty);
+            });
         }
 
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
         {
-            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
+            return OnCodex(
+                api => api.UploadAsync(fileStream),
+                CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
-        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
+        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
-            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+            var fileResponse = OnCodex(
+                api => api.DownloadNetworkAsync(contentId),
+                CreateRetryConfig(nameof(DownloadFile), onFailure));
+
+            if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
+            return fileResponse.Stream;
         }
 
-        public string UploadFile(FileStream fileStream)
+        public LocalDatasetList LocalFiles()
         {
-            return Http().HttpPostStream("data", fileStream);
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
         }
 
-        public Stream DownloadFile(string contentId)
+        public StorageAvailability SalesAvailability(StorageAvailability request)
         {
-            return Http().HttpGetStream("data/" + contentId + "/network");
+            var body = mapper.Map(request);
+            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            return mapper.Map(read);
         }
 
-        public CodexLocalDataResponse[] LocalFiles()
+        public string RequestStorage(StoragePurchaseRequest request)
         {
-            return Http().HttpGetJson<CodexLocalDataResponse[]>("data");
+            var body = mapper.Map(request);
+            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        public CodexSpace Space()
         {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+            var space = OnCodex<Space>(api => api.SpaceAsync());
+            return mapper.Map(space);
         }
 
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
-        }
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+                }, nameof(GetPurchaseStatus));
+            });
 
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+            // TODO: current getpurchase api does not line up with its openapi spec.
+            // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
         }
 
         public string GetName()
@@ -107,43 +132,137 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
-        private IHttp Http()
+        public void LogDiskSpace(string msg)
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            try
+            {
+                var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
+                Log($"{msg} - Disk info: {diskInfo}");
+            }
+            catch (Exception e)
+            {
+                Log("Failed to get disk info: " + e);
+            }
         }
 
-        private IHttp LongHttp()
+        public void DeleteRepoFolder()
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+            try
+            {
+                var containerNumber = Container.Containers.First().Recipe.Number;
+                var dataDir = $"datadir{containerNumber}";
+                var workflow = tools.CreateWorkflow();
+                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
+                Log("Deleted repo folder.");
+            }
+            catch (Exception e)
+            {
+                Log("Unable to delete repo folder: " + e);
+            }
+        }
+
+        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
+        {
+            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action));
+            return result;
+        }
+
+        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
+        {
+            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            return result;
+        }
+
+        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
+        {
+            var address = GetAddress();
+            var api = new CodexApi(client);
+            api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return CrashCheck(() => Time.Wait(action(api)));
+        }
+
+        private T CrashCheck<T>(Func<T> action)
+        {
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                CrashWatcher.HasContainerCrashed();
+            }
+        }
+
+        private IEndpoint GetEndpoint()
+        {
+            return tools
+                .CreateHttp(CheckContainerCrashed)
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
         private Address GetAddress()
         {
-            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
         {
-            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
         }
 
-        public void Log(Stream crashLog)
+        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
         {
-            var log = tools.GetLog();
-            var file = log.CreateSubfile();
-            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
-            file.Write($"Container Crash Log for {Container.Name}.");
+            var timeSet = tools.TimeSet;
 
-            using var reader = new StreamReader(crashLog);
-            var line = reader.ReadLine();
-            while (line != null)
+            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
             {
-                file.Write(line);
-                line = reader.ReadLine();
+                onFailure(failure);
+                Investigate(failure, timeSet);
+            });
+        }
+
+        private void Investigate(Failure failure, ITimeSet timeSet)
+        {
+            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
+                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
+                $"Checking if node responds to debug/info...");
+
+            LogDiskSpace("After retry failure");
+
+            try
+            {
+                var debugInfo = GetDebugInfo();
+                if (string.IsNullOrEmpty(debugInfo.Spr))
+                {
+                    Log("Did not get value debug/info response.");
+                    Throw(failure);
+                }
+                else
+                {
+                    Log("Got valid response from debug/info.");
+                }
+            }
+            catch (Exception ex)
+            {
+                Log("Got exception from debug/info call: " + ex);
+                Throw(failure);
             }
 
-            log.Log("Crash log successfully downloaded.");
-            hasContainerCrashed = true;
+            if (failure.Duration < timeSet.HttpCallTimeout())
+            {
+                Log("Retry failed within HTTP timeout duration.");
+                Throw(failure);
+            }
+        }
+
+        private void Throw(Failure failure)
+        {
+            throw failure.Exception;
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"{GetName()} {msg}");
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -1,195 +0,0 @@
-using Newtonsoft.Json;
-
-namespace CodexPlugin
-{
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = Array.Empty<string>();
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public string[] announceAddresses { get; set; } = Array.Empty<string>();
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugFutures
-    {
-        public int futures { get; set; }
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public string[] addresses { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexDebugThresholdBreaches
-    {
-        public string[] breaches { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set; }
-    }
-
-    public class CodexStoragePurchase
-    {
-        public string state { get; set; } = string.Empty;
-        public string error { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponse
-    {
-        public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
-        public int taskQueue { get; set; }
-        public int pendingBlocks { get; set; }
-
-        public override string ToString()
-        {
-            if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
-
-            return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeer
-    {
-        public string peerid { get; set; } = string.Empty;
-        public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
-        public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
-        public int exchanged { get; set; }
-
-        public override string ToString()
-        {
-            return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerHasBlock
-    {
-        public string cid { get; set; } = string.Empty;
-        public bool have { get; set; }
-        public string price { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerWant
-    {
-        public string block { get; set; } = string.Empty;
-        public int priority { get; set; }
-        public bool cancel { get; set; }
-        public string wantType { get; set; } = string.Empty;
-        public bool sendDontHave { get; set; }
-    }
-
-    public class CodexDebugRepoStoreResponse
-    {
-        public string cid { get; set; } = string.Empty; 
-    }
-
-    public class CodexLocalData
-    {
-        public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
-        {
-            Cid = cid;
-            Manifest = manifest;
-        }
-
-        public ContentId Cid { get; }
-        public CodexLocalDataManifestResponse Manifest { get; }
-    }
-
-    public class CodexLocalDataResponse
-    {
-        public string cid { get; set; } = string.Empty;
-        public CodexLocalDataManifestResponse manifest { get; set; } = new();
-    }
-
-    public class CodexLocalDataManifestResponse
-    {
-        public string rootHash { get; set; } = string.Empty;
-        public int originalBytes { get; set; }
-        public int blockSize { get; set; }
-        public bool @protected { get; set; }
-    }
-}

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,9 +7,8 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-471ebb2-dist-tests";
 
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-e4ddb94-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -106,13 +105,14 @@ namespace CodexPlugin
                 AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
                 AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
 
+                var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
+
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                var mStart = marketplaceStarter.Start();
-                AddEnvVar("PRIV_KEY", mStart.PrivateKey);
-                Additional(mStart);
+                var account = marketplaceSetup.EthAccountSetup.GetNew();
+                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                Additional(account);
 
-                var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
                 SetCommandOverride(marketplaceSetup);
                 if (marketplaceSetup.IsValidator)
                 {
@@ -120,7 +120,7 @@ namespace CodexPlugin
                 }
             }
 
-            if(!string.IsNullOrEmpty(config.NameOverride))
+            if (!string.IsNullOrEmpty(config.NameOverride))
             {
                 AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
@@ -160,7 +160,7 @@ namespace CodexPlugin
 
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)
         {
-            if (config.StorageQuota != null) return config.StorageQuota;
+            if (config.StorageQuota != null) return config.StorageQuota.Multiply(1.2);
             // Default Codex quota: 8 Gb, using +20% to be safe.
             return 8.GB().Multiply(1.2);
         }

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
     public class CodexDeployment
     {
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
-            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
-            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
+            RunningPod? discordBotContainer, DeploymentMetadata metadata,
             String id)
         {
             Id = id;
@@ -24,21 +24,21 @@ namespace CodexPlugin
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningContainers? PrometheusContainer { get; }
-        public RunningContainers? DiscordBotContainer { get; }
+        public RunningPod? PrometheusContainer { get; }
+        public RunningPod? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningContainers containers, CodexDebugResponse info)
+        public CodexInstance(RunningPod pod, DebugInfo info)
         {
-            Containers = containers;
+            Pod = pod;
             Info = info;
         }
 
-        public RunningContainers Containers { get; }
-        public CodexDebugResponse Info { get; }
+        public RunningPod Pod { get; }
+        public DebugInfo Info { get; }
     }
 
     public class DeploymentMetadata

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -0,0 +1,105 @@
+using System.Globalization;
+
+namespace CodexPlugin
+{
+    public class CodexLogLine
+    {
+        public static CodexLogLine? Parse(string line)
+        {
+            try
+            {
+                if (string.IsNullOrEmpty(line) ||
+                    line.Length < 34 ||
+                    line[3] != ' ' ||
+                    line[33] != ' ') return null;
+
+                line = line.Replace(Environment.NewLine, string.Empty);
+
+                var level = line.Substring(0, 3);
+                var dtLine = line.Substring(4, 23);
+
+                var firstEqualSign = line.IndexOf('=');
+                var msgStart = 34;
+                var msgEnd = line.Substring(0, firstEqualSign).LastIndexOf(' ');
+                var msg = line.Substring(msgStart, msgEnd - msgStart).Trim();
+                var attrsLine = line.Substring(msgEnd);
+
+                var attrs = SplitAttrs(attrsLine);
+
+                var format = "yyyy-MM-dd HH:mm:ss.fff";
+                var dt = DateTime.ParseExact(dtLine, format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime();
+
+                return new CodexLogLine()
+                {
+                    LogLevel = level,
+                    TimestampUtc = dt,
+                    Message = msg,
+                    Attributes = attrs
+                };
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public string LogLevel { get; set; } = string.Empty;
+        public DateTime TimestampUtc { get; set; }
+        public string Message { get; set; } = string.Empty;
+        public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
+
+        /// <summary>
+        /// After too much time spent cursing at regexes, here's what I got:
+        /// Parses input string into 'key=value' pair, considerate of quoted (") values.
+        /// </summary>
+        private static Dictionary<string, string> SplitAttrs(string input)
+        {
+            input += " ";
+            var result = new Dictionary<string, string>();
+
+            var key = string.Empty;
+            var value = string.Empty;
+            var mode = 1;
+            var inQuote = false;
+
+            foreach (var c in input)
+            {
+                if (mode == 1)
+                {
+                    if (c == '=') mode = 2;
+                    else if (c == ' ')
+                    {
+                        if (string.IsNullOrEmpty(key)) continue;
+                        else
+                        {
+                            result.Add(key, string.Empty);
+                            key = string.Empty;
+                            value = string.Empty;
+                        }
+                    }
+                    else key += c;
+                }
+                else if (mode == 2)
+                {
+                    if (c == ' ' && !inQuote)
+                    {
+                        result.Add(key, value);
+                        key = string.Empty;
+                        value = string.Empty;
+                        mode = 1;
+                    }
+                    else if (c == '\"')
+                    {
+                        inQuote = !inQuote;
+                    }
+                    else
+                    {
+                        value += c;
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -12,102 +12,118 @@ namespace CodexPlugin
     public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
     {
         string GetName();
-        CodexDebugResponse GetDebugInfo();
-        CodexDebugPeerResponse GetDebugPeer(string peerId);
-        // These debug methods are not available in master-line Codex. Use only for custom builds.
-        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
-        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
+        DebugInfo GetDebugInfo();
+        DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
+        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        CodexLocalData[] LocalFiles();
+        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDatasetList LocalFiles();
+        CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
-        CodexDebugVersionResponse Version { get; }
+        DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
+        EthAccount EthAccount { get; }
+
+        /// <summary>
+        /// Warning! The node is not usable after this.
+        /// TODO: Replace with delete-blocks debug call once available in Codex.
+        /// </summary>
+        void DeleteRepoFolder();
         void Stop(bool waitTillStopped);
     }
 
     public class CodexNode : ICodexNode
     {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
-        private readonly EthAddress? ethAddress;
+        private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
         {
             this.tools = tools;
-            this.ethAddress = ethAddress;
+            this.ethAccount = ethAccount;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
+            Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
         }
 
-        public RunningContainer Container { get { return CodexAccess.Container; } }
+        public RunningPod Pod { get { return CodexAccess.Container; } }
+        
+        public RunningContainer Container { get { return Pod.Containers.Single(); } }
         public CodexAccess CodexAccess { get; }
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
         public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
+
         public IMetricsScrapeTarget MetricsScrapeTarget
         {
             get
             {
-                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
             }
         }
+
         public EthAddress EthAddress 
         {
             get
             {
-                if (ethAddress == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
-                return ethAddress;
+                EnsureMarketplace();
+                return ethAccount!.EthAddress;
+            }
+        }
+
+        public EthAccount EthAccount
+        {
+            get
+            {
+                EnsureMarketplace();
+                return ethAccount!;
             }
         }
 
         public string GetName()
         {
-            return CodexAccess.Container.Name;
+            return Container.Name;
         }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
-            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
+            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
             return debugInfo;
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
-        {
-            return CodexAccess.GetDebugBlockExchange();
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return CodexAccess.GetDebugRepoStore();
-        }
-
         public ContentId UploadFile(TrackedFile file)
         {
+            return UploadFile(file, DoNothing);
+        }
+
+        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            CodexAccess.LogDiskSpace("Before upload");
+
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
             var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream);
+                return CodexAccess.UploadFile(fileStream, onFailure);
             });
 
             var response = measurement.Value;
@@ -117,23 +133,35 @@ namespace CodexPlugin
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
             Log($"Uploaded file. Received contentId: '{response}'.");
+            CodexAccess.LogDiskSpace("After upload");
+
             return new ContentId(response);
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContent(contentId, DoNothing, fileLabel);
+        }
+
+        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
             transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }
 
-        public CodexLocalData[] LocalFiles()
+        public LocalDatasetList LocalFiles()
         {
-            return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray();
+            return CodexAccess.LocalFiles();
+        }
+
+        public CodexSpace Space()
+        {
+            return CodexAccess.Space();
         }
 
         public void ConnectToPeer(ICodexNode node)
@@ -142,9 +170,8 @@ namespace CodexPlugin
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
 
-            FrameworkAssert.That(response == SuccessfullyConnectedMessage, "Unable to connect codex nodes.");
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
@@ -153,51 +180,55 @@ namespace CodexPlugin
             return CodexAccess.GetPodInfo();
         }
 
+        public void DeleteRepoFolder()
+        {
+            CodexAccess.DeleteRepoFolder();
+        }
+
         public void Stop(bool waitTillStopped)
         {
-            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-                "available for codex-nodes in groups of 1.");
-
-            Group.BringOffline(waitTillStopped);
+            Log("Stopping...");
+            CrashWatcher.Stop();
+            Group.Stop(this, waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
         {
             var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.id;
+            var nodePeerId = debugInfo.Id;
             var nodeName = CodexAccess.Container.Name;
 
-            if (!debugInfo.codex.IsValid())
+            if (!debugInfo.Version.IsValid())
             {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
             var log = tools.GetLog();
             log.AddStringReplace(nodePeerId, nodeName);
-            log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
+            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
+            Version = debugInfo.Version;
         }
 
-        private string GetPeerMultiAddress(CodexNode peer, CodexDebugResponse peerInfo)
+        private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Container);
+            var podInfo = workflow.GetPodInfo(peer.Pod);
 
-            return multiAddress.Replace("0.0.0.0", podInfo.Ip);
+            return peerInfo.Addrs.Select(a => a
+                .Replace("0.0.0.0", podInfo.Ip))
+                .ToArray();
         }
 
-        private void DownloadToFile(string contentId, TrackedFile file)
+        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
+            CodexAccess.LogDiskSpace("Before download");
+
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
                 downloadStream.CopyTo(fileStream);
             }
             catch
@@ -205,31 +236,22 @@ namespace CodexPlugin
                 Log($"Failed to download file '{contentId}'.");
                 throw;
             }
+
+            CodexAccess.LogDiskSpace("After download");
+        }
+
+        private void EnsureMarketplace()
+        {
+            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
         }
 
         private void Log(string msg)
         {
             tools.GetLog().Log($"{GetName()}: {msg}");
         }
-    }
 
-    public class ContentId
-    {
-        public ContentId(string id)
+        private void DoNothing(Failure failure)
         {
-            Id = id;
-        }
-
-        public string Id { get; }
-
-        public override bool Equals(object? obj)
-        {
-            return obj is ContentId id && Id == id.Id;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(Id);
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -22,22 +22,22 @@ namespace CodexPlugin
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
-            var ethAddress = GetEthAddress(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAddress);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAddress);
+            var ethAccount = GetEthAccount(access);
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
+            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAddress? ethAddress)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
         {
-            if (ethAddress == null) return new MarketplaceUnavailable();
+            if (ethAccount == null) return new MarketplaceUnavailable();
             return new MarketplaceAccess(tools.GetLog(), codexAccess);
         }
 
-        private EthAddress? GetEthAddress(CodexAccess access)
+        private EthAccount? GetEthAccount(CodexAccess access)
         {
-            var mStart = access.Container.Recipe.Additionals.Get<MarketplaceStartResults>();
-            if (mStart == null) return null;
-            return mStart.EthAddress;
+            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
+            if (ethAccount == null) return null;
+            return ethAccount;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -15,12 +15,12 @@ namespace CodexPlugin
     {
         private readonly CodexStarter starter;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.starter = starter;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
+            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Version = new DebugInfoVersion();
         }
 
         public ICodexNode this[int index]
@@ -39,9 +39,16 @@ namespace CodexPlugin
             Containers = null!;
         }
 
-        public RunningContainers[] Containers { get; private set; }
+        public void Stop(CodexNode node, bool waitTillStopped)
+        {
+            starter.Stop(node.Pod, waitTillStopped);
+            Nodes = Nodes.Where(n => n != node).ToArray();
+            Containers = Containers.Where(c => c != node.Pod).ToArray();
+        }
+
+        public RunningPod[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
 
         public IEnumerator<ICodexNode> GetEnumerator()
@@ -65,7 +72,7 @@ namespace CodexPlugin
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            if (!versionResponses.All(v => v.Version == first.Version && v.Revision == first.Revision))
             {
                 throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
                     string.Join(",", versionResponses.Select(v => v.ToString())));
@@ -74,9 +81,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
+        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c);
+            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
             var access = new CodexAccess(tools, c, watcher);
             return factory.CreateOnlineCodexNode(access, this);
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,4 +1,4 @@
-using Core;
+using Core;
 using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
@@ -32,13 +32,13 @@ namespace CodexPlugin
         {
         }
 
-        public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
             return codexStarter.WrapCodexContainers(coreInterface, containers);
@@ -52,8 +52,8 @@ namespace CodexPlugin
             var mconfig = codexSetup.MarketplaceConfig;
             foreach (var node in result)
             {
-                mconfig.GethNode.SendEth(node, mconfig.InitialEth);
-                mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens);
+                mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
+                mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
             }
         }
 

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -7,7 +7,23 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>
@@ -18,4 +34,8 @@
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />
   </ItemGroup>
 
+  <Target Name="PreBuild" BeforeTargets="PreBuildEvent">
+    <Exec Command="dotnet run --project $(ProjectDir)\..\CodexPluginPrebuild" />
+  </Target>
+
 </Project>

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -17,8 +17,7 @@ namespace CodexPlugin
         ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
         ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens);
-        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, Action<IMarketplaceSetup> marketplaceSetup);
+        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup);
         /// <summary>
         /// Provides an invalid proof every N proofs
         /// </summary>
@@ -28,6 +27,8 @@ namespace CodexPlugin
 
     public interface IMarketplaceSetup
     {
+        IMarketplaceSetup WithInitial(Ether eth, TestToken tokens);
+        IMarketplaceSetup WithAccount(EthAccount account);
         IMarketplaceSetup AsStorageNode();
         IMarketplaceSetup AsValidator();
     }
@@ -49,7 +50,9 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
+        public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn;
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -75,7 +78,7 @@ namespace CodexPlugin
 
         public ICodexSetup WithBootstrapNode(ICodexNode node)
         {
-            BootstrapSpr = node.GetDebugInfo().spr;
+            BootstrapSpr = node.GetDebugInfo().Spr;
             return this;
         }
 
@@ -122,17 +125,12 @@ namespace CodexPlugin
             return this;
         }
 
-        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens)
-        {
-            return EnableMarketplace(gethNode, codexContracts, initialEth, initialTokens, s => { });
-        }
-
-        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, Action<IMarketplaceSetup> marketplaceSetup)
+        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup)
         {
             var ms = new MarketplaceSetup();
             marketplaceSetup(ms);
 
-            MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts, initialEth, initialTokens);
+            MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts);
             return this;
         }
 
@@ -169,6 +167,9 @@ namespace CodexPlugin
     {
         public bool IsStorageNode { get; private set; }
         public bool IsValidator { get; private set; }
+        public Ether InitialEth { get; private set; } = 0.Eth();
+        public TestToken InitialTestTokens { get; private set; } = 0.Tst();
+        public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup();
 
         public IMarketplaceSetup AsStorageNode()
         {
@@ -182,14 +183,59 @@ namespace CodexPlugin
             return this;
         }
 
+        public IMarketplaceSetup WithAccount(EthAccount account)
+        {
+            EthAccountSetup.Pin(account);
+            return this;
+        }
+
+        public IMarketplaceSetup WithInitial(Ether eth, TestToken tokens)
+        {
+            InitialEth = eth;
+            InitialTestTokens = tokens;
+            return this;
+        }
+
         public override string ToString()
         {
             var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
             result += IsStorageNode ? "(storageNode)" : "()";
-            result += IsValidator ? "(validator)" : "()";
-            result += "]";
+            result += IsValidator ? "(validator)" : "() ";
+            result += $"Address: '{EthAccountSetup}' ";
+            result += $"{InitialEth.Eth} / {InitialTestTokens}";
+            result += "] ";
             return result;
         }
     }
 
+    public class EthAccountSetup
+    {
+        private readonly List<EthAccount> accounts = new List<EthAccount>();
+        private bool pinned = false;
+
+        public void Pin(EthAccount account)
+        {
+            accounts.Add(account);
+            pinned = true;
+        }
+
+        public EthAccount GetNew()
+        {
+            if (pinned) return accounts.Last();
+
+            var a = EthAccount.GenerateNew();
+            accounts.Add(a);
+            return a;
+        }
+
+        public EthAccount[] GetAll()
+        {
+            return accounts.ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join(",", accounts.Select(a => a.ToString()).ToArray());
+        }
+    }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -9,14 +9,17 @@ namespace CodexPlugin
     {
         private readonly IPluginTools pluginTools;
         private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
-        private CodexDebugVersionResponse? versionResponse;
+        private readonly ApiChecker apiChecker;
+        private DebugInfoVersion? versionResponse;
 
         public CodexStarter(IPluginTools pluginTools)
         {
             this.pluginTools = pluginTools;
+
+            apiChecker = new ApiChecker(pluginTools);
         }
 
-        public RunningContainers[] BringOnline(CodexSetup codexSetup)
+        public RunningPod[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -25,18 +28,20 @@ namespace CodexPlugin
 
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
+            apiChecker.CheckCompatibility(containers);
+
             foreach (var rc in containers)
             {
                 var podInfo = GetPodInfo(rc);
-                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' PodLabel: '{c.RunningPod.StartResult.Deployment.PodLabel}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();
 
             return containers;
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             var codexNodeFactory = new CodexNodeFactory(pluginTools);
 
@@ -60,15 +65,23 @@ namespace CodexPlugin
             Log("Stopped.");
         }
 
+        public void Stop(RunningPod pod, bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            var workflow = pluginTools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            Log("Stopped.");
+        }
+
         public string GetCodexId()
         {
-            if (versionResponse != null) return versionResponse.version;
+            if (versionResponse != null) return versionResponse.Version;
             return recipe.Image;
         }
 
         public string GetCodexRevision()
         {
-            if (versionResponse != null) return versionResponse.revision;
+            if (versionResponse != null) return versionResponse.Revision;
             return "unknown";
         }
 
@@ -80,24 +93,27 @@ namespace CodexPlugin
             return startupConfig;
         }
 
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
+        private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var result = new List<RunningContainers>();
+            var futureContainers = new List<FutureContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
+                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-            return result.ToArray();
+
+            return futureContainers
+                .Select(f => f.WaitForOnline())
+                .ToArray();
         }
 
-        private PodInfo GetPodInfo(RunningContainers rc)
+        private PodInfo GetPodInfo(RunningPod rc)
         {
             var workflow = pluginTools.CreateWorkflow();
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
 
@@ -114,10 +130,10 @@ namespace CodexPlugin
             return group;
         }
 
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
+        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
         {
             Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
+            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
         }
 
         private void LogSeparator()

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -73,12 +73,18 @@ namespace CodexPlugin
                     "contracts",
                     "clock"
                 };
+                var jsonSerializeTopics = new[]
+                {
+                    "serde",
+                    "json",
+                    "serialization"
+                };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
-                    // Contract clock is always set to warn. It logs a trace every second.
-                    $"{CodexLogLevel.Warn.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
+                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
+                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -0,0 +1,122 @@
+using Newtonsoft.Json;
+using Utils;
+
+namespace CodexPlugin
+{
+    public class DebugInfo
+    {
+        public string[] Addrs { get; set; } = Array.Empty<string>();
+        public string Spr { get; set; } = string.Empty;
+        public string Id { get; set; } = string.Empty;
+        public string[] AnnounceAddresses { get; set; } = Array.Empty<string>();
+        public DebugInfoVersion Version { get; set; } = new();
+        public DebugInfoTable Table { get; set; } = new();
+    }
+
+    public class DebugInfoVersion
+    {
+        public string Version { get; set; } = string.Empty;
+        public string Revision { get; set; } = string.Empty;
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(Version) && !string.IsNullOrEmpty(Revision);
+        }
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+    }
+
+    public class DebugInfoTable
+    {
+        public DebugInfoTableNode LocalNode { get; set; } = new();
+        public DebugInfoTableNode[] Nodes { get; set; } = Array.Empty<DebugInfoTableNode>();
+    }
+
+    public class DebugInfoTableNode
+    {
+        public string NodeId { get; set; } = string.Empty;
+        public string PeerId { get; set; } = string.Empty;
+        public string Record { get; set; } = string.Empty;
+        public string Address { get; set; } = string.Empty;
+        public bool Seen { get; set; }
+    }
+
+    public class DebugPeer
+    {
+        public bool IsPeerFound { get; set; }
+        public string PeerId { get; set; } = string.Empty;
+        public string[] Addresses { get; set; } = Array.Empty<string>();
+    }
+
+    public class LocalDatasetList
+    {
+        public LocalDataset[] Content { get; set; } = Array.Empty<LocalDataset>();
+    } 
+
+    public class LocalDataset
+    {
+        public ContentId Cid { get; set; } = new();
+        public Manifest Manifest { get; set; } = new();
+    }
+
+    public class Manifest
+    {
+        public string RootHash { get; set; } = string.Empty;
+        public ByteSize OriginalBytes { get; set; } = ByteSize.Zero;
+        public ByteSize BlockSize { get; set; } = ByteSize.Zero;
+        public bool Protected { get; set; }
+    }
+
+    public class SalesRequestStorageRequest
+    {
+        public string Duration { get; set; } = string.Empty;
+        public string ProofProbability { get; set; } = string.Empty;
+        public string Reward { get; set; } = string.Empty;
+        public string Collateral { get; set; } = string.Empty;
+        public string? Expiry { get; set; }
+        public uint? Nodes { get; set; }
+        public uint? Tolerance { get; set; }
+    }
+
+    public class ContentId
+    {
+        public ContentId()
+        {
+            Id = string.Empty;
+        }
+
+        public ContentId(string id)
+        {
+            Id = id;
+        }
+
+        public string Id { get; }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ContentId id && Id == id.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+    }
+
+    public class CodexSpace
+    {
+        public long TotalBlocks { get; set; }
+        public long QuotaMaxBytes { get; set; }
+        public long QuotaUsedBytes { get; set; }
+        public long QuotaReservedBytes { get; set; }
+        public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);
         }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -0,0 +1,226 @@
+using CodexContractsPlugin;
+using CodexOpenApi;
+using Newtonsoft.Json.Linq;
+using System.Numerics;
+using Utils;
+
+namespace CodexPlugin
+{
+    public class Mapper
+    {
+        public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo)
+        {
+            return new DebugInfo
+            {
+                Id = debugInfo.Id,
+                Spr = debugInfo.Spr,
+                Addrs = debugInfo.Addrs.ToArray(),
+                AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
+                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
+                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+            };
+        }
+
+        public LocalDatasetList Map(CodexOpenApi.DataList dataList)
+        {
+            return new LocalDatasetList
+            {
+                Content = dataList.Content.Select(Map).ToArray()
+            };
+        }
+
+        public LocalDataset Map(CodexOpenApi.DataItem dataItem)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(dataItem.Cid),
+                Manifest = MapManifest(dataItem.Manifest)
+            };
+        }
+
+        public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
+        {
+            return new CodexOpenApi.SalesAvailabilityCREATE
+            {
+                Duration = ToDecInt(availability.MaxDuration.TotalSeconds),
+                MinPrice = ToDecInt(availability.MinPriceForTotalSpace),
+                MaxCollateral = ToDecInt(availability.MaxCollateral),
+                TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes)
+            };
+        }
+
+        public CodexOpenApi.StorageRequestCreation Map(StoragePurchaseRequest purchase)
+        {
+            return new CodexOpenApi.StorageRequestCreation
+            {
+                Duration = ToDecInt(purchase.Duration.TotalSeconds),
+                ProofProbability = ToDecInt(purchase.ProofProbability),
+                Reward = ToDecInt(purchase.PricePerSlotPerSecond),
+                Collateral = ToDecInt(purchase.RequiredCollateral),
+                Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
+                Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
+                Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
+            };
+        }
+
+        // TODO: Fix openapi spec for this call.
+        //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        //{
+        //    return new StoragePurchase(Map(purchase.Request))
+        //    {
+        //        State = purchase.State,
+        //        Error = purchase.Error
+        //    };
+        //}
+
+        //public StorageRequest Map(CodexOpenApi.StorageRequest request)
+        //{
+        //    return new StorageRequest(Map(request.Ask), Map(request.Content))
+        //    {
+        //        Id = request.Id,
+        //        Client = request.Client,
+        //        Expiry = TimeSpan.FromSeconds(Convert.ToInt64(request.Expiry)),
+        //        Nonce = request.Nonce
+        //    };
+        //}
+
+        //public StorageAsk Map(CodexOpenApi.StorageAsk ask)
+        //{
+        //    return new StorageAsk
+        //    {
+        //        Duration = TimeSpan.FromSeconds(Convert.ToInt64(ask.Duration)),
+        //        MaxSlotLoss = ask.MaxSlotLoss,
+        //        ProofProbability = ask.ProofProbability,
+        //        Reward = Convert.ToDecimal(ask.Reward).TstWei(),
+        //        Slots = ask.Slots,
+        //        SlotSize = new ByteSize(Convert.ToInt64(ask.SlotSize))
+        //    };
+        //}
+
+        //public StorageContent Map(CodexOpenApi.Content content)
+        //{
+        //    return new StorageContent
+        //    {
+        //        Cid = content.Cid
+        //    };
+        //}
+
+        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
+        {
+            return new StorageAvailability(
+                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
+                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
+                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
+                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
+            )
+            {
+                Id = read.Id
+            };
+        }
+
+        public CodexSpace Map(Space space)
+        {
+            return new CodexSpace
+            {
+                QuotaMaxBytes = space.QuotaMaxBytes,
+                QuotaReservedBytes = space.QuotaReservedBytes,
+                QuotaUsedBytes = space.QuotaUsedBytes,
+                TotalBlocks = space.TotalBlocks
+            };
+        }
+
+        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        {
+            return new DebugInfoVersion
+            {
+                Version = StringOrEmpty(obj, "version"),
+                Revision = StringOrEmpty(obj, "revision")
+            };
+        }
+
+        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        {
+            return new DebugInfoTable
+            {
+                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
+                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+            };
+        }
+
+        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        {
+            var obj = token as JObject;
+            if (obj == null) return new DebugInfoTableNode();
+
+            return new DebugInfoTableNode
+            {
+                Address = StringOrEmpty(obj, "address"),
+                NodeId = StringOrEmpty(obj, "nodeId"),
+                PeerId = StringOrEmpty(obj, "peerId"),
+                Record = StringOrEmpty(obj, "record"),
+                Seen = Bool(obj, "seen")
+            };
+        }
+
+        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        {
+            if (nodes == null || nodes.Count == 0)
+            {
+                return new DebugInfoTableNode[0];
+            }
+
+            return nodes.Select(MapDebugInfoTableNode).ToArray();
+        }
+
+        private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
+        {
+            return new Manifest
+            {
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
+                RootHash = manifest.RootHash,
+                Protected = manifest.Protected
+            };
+        }
+
+        private JArray JArray(IDictionary<string, object> map, string name)
+        {
+            return (JArray)map[name];
+        }
+
+        private JObject JObject(IDictionary<string, object> map, string name)
+        {
+            return (JObject)map[name];
+        }
+
+        private string StringOrEmpty(JObject obj, string name)
+        {
+            if (obj.TryGetValue(name, out var token))
+            {
+                var str = (string?)token;
+                if (!string.IsNullOrEmpty(str)) return str;
+            }
+            return string.Empty;
+        }
+
+        private bool Bool(JObject obj, string name)
+        {
+            if (obj.TryGetValue(name, out var token))
+            {
+                return (bool)token;
+            }
+            return false;
+        }
+
+        private string ToDecInt(double d)
+        {
+            var i = new BigInteger(d);
+            return i.ToString("D");
+        }
+
+        private string ToDecInt(TestToken t)
+        {
+            return t.TstWei.ToString("D");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,5 +1,4 @@
 using Logging;
-using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
@@ -7,7 +6,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
-        StoragePurchaseContract RequestStorage(StoragePurchase purchase);
+        IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -21,14 +20,14 @@ namespace CodexPlugin
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchase purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
-            var request = purchase.ToApiRequest();
 
-            var response = codexAccess.RequestStorage(request, purchase.ContentId.Id);
+            var response = codexAccess.RequestStorage(purchase);
 
-            if (response == "Purchasing not available" || 
+            if (string.IsNullOrEmpty(response) ||
+                response == "Purchasing not available" ||
                 response == "Expiry required" ||
                 response == "Expiry needs to be in future" ||
                 response == "Expiry has to be before the request's end (now + duration)")
@@ -38,24 +37,25 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            return new StoragePurchaseContract(log, codexAccess, response, purchase);
+            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase);
+            contract.WaitForStorageContractSubmitted();
+            return contract;
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
         {
             availability.Log(log);
-            var request = availability.ToApiRequest();
 
-            var response = codexAccess.SalesAvailability(request);
+            var response = codexAccess.SalesAvailability(availability);
 
-            Log($"Storage successfully made available. Id: {response.id}");
+            Log($"Storage successfully made available. Id: {response.Id}");
 
-            return response.id;
+            return response.Id;
         }
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Name} {msg}");
+            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
         }
     }
 
@@ -67,7 +67,7 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchase purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             Unavailable();
             throw new NotImplementedException();
@@ -79,78 +79,4 @@ namespace CodexPlugin
             throw new InvalidOperationException();
         }
     }
-
-    public class StoragePurchaseContract
-    {
-        private readonly ILog log;
-        private readonly CodexAccess codexAccess;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10);
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchase purchase)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            Purchase = purchase;
-        }
-
-        public string PurchaseId { get; }
-        public StoragePurchase Purchase { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            var timeout = Purchase.Expiry + gracePeriod;
-
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.state != lastState)
-                {
-                    lastState = purchaseStatus.state;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    FrameworkAssert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-    }
 }

ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs

@@ -5,19 +5,15 @@ namespace CodexPlugin
 {
     public class MarketplaceInitialConfig
     {
-        public MarketplaceInitialConfig(MarketplaceSetup marketplaceSetup, IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens)
+        public MarketplaceInitialConfig(MarketplaceSetup marketplaceSetup, IGethNode gethNode, ICodexContracts codexContracts)
         {
             MarketplaceSetup = marketplaceSetup;
             GethNode = gethNode;
             CodexContracts = codexContracts;
-            InitialEth = initialEth;
-            InitialTokens = initialTokens;
         }
 
         public MarketplaceSetup MarketplaceSetup { get; }
         public IGethNode GethNode { get; }
         public ICodexContracts CodexContracts { get; }
-        public Ether InitialEth { get; }
-        public TestToken InitialTokens { get; }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs

@@ -1,17 +0,0 @@
-using GethPlugin;
-
-namespace CodexPlugin
-{
-    [Serializable]
-    public class MarketplaceStartResults
-    {
-        public MarketplaceStartResults(EthAddress ethAddress, string privateKey)
-        {
-            EthAddress = ethAddress;
-            PrivateKey = privateKey;
-        }
-
-        public EthAddress EthAddress { get; }
-        public string PrivateKey { get; }
-    }
-}

ProjectPlugins/CodexPlugin/MarketplaceStarter.cs

@@ -1,19 +0,0 @@
-using GethPlugin;
-using Nethereum.Hex.HexConvertors.Extensions;
-using Nethereum.Web3.Accounts;
-
-namespace CodexPlugin
-{
-    public class MarketplaceStarter
-    {
-        public MarketplaceStartResults Start()
-        {
-            var ecKey = Nethereum.Signer.EthECKey.GenerateKey();
-            var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex();
-            var account = new Account(privateKey);
-            var ethAddress = new EthAddress(account.Address);
-
-            return new MarketplaceStartResults(ethAddress, account.PrivateKey);
-        }
-    }
-}

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,40 +1,27 @@
 using CodexContractsPlugin;
+using CodexOpenApi;
 using Logging;
-using System.Numerics;
+using System.Data;
 using Utils;
 
 namespace CodexPlugin
 {
-    public class StoragePurchase : MarketplaceType
+    public class StoragePurchaseRequest
     {
-        public StoragePurchase(ContentId cid)
+        public StoragePurchaseRequest(ContentId cid)
         {
             ContentId = cid;
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TestTokens();
-        public TestToken RequiredCollateral { get; set; } = 1.TestTokens();
+        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
+        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }
         public TimeSpan Duration { get; set; }
         public TimeSpan Expiry { get; set; }
 
-        public CodexSalesRequestStorageRequest ToApiRequest()
-        {
-            return new CodexSalesRequestStorageRequest
-            {
-                duration = ToDecInt(Duration.TotalSeconds),
-                proofProbability = ToDecInt(ProofProbability),
-                reward = ToDecInt(PricePerSlotPerSecond),
-                collateral = ToDecInt(RequiredCollateral),
-                expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + Expiry.TotalSeconds),
-                nodes = MinRequiredNumberOfNodes,
-                tolerance = NodeFailureTolerance
-            };
-        }
-
         public void Log(ILog log)
         {
             log.Log($"Requesting storage for: {ContentId.Id}... (" +
@@ -48,7 +35,41 @@ namespace CodexPlugin
         }
     }
 
-    public class StorageAvailability : MarketplaceType
+    public class StoragePurchase
+    {
+        public string State { get; set; } = string.Empty;
+        public string Error { get; set; } = string.Empty;
+        public StorageRequest Request { get; set; } = null!;
+    }
+
+    public class StorageRequest
+    {
+        public string Id { get; set; } = string.Empty;
+        public string Client { get; set; } = string.Empty;
+        public StorageAsk Ask { get; set; } = null!;
+        public StorageContent Content { get; set; } = null!;
+        public string Expiry { get; set; } = string.Empty;
+        public string Nonce { get; set; } = string.Empty;
+    }
+
+    public class StorageAsk
+    {
+        public int Slots { get; set; }
+        public string SlotSize { get; set; } = string.Empty;
+        public string Duration { get; set; } = string.Empty;
+        public string ProofProbability { get; set; } = string.Empty;
+        public string Reward { get; set; } = string.Empty;
+        public int MaxSlotLoss { get; set; }
+    }
+
+    public class StorageContent
+    {
+        public string Cid { get; set; } = string.Empty;
+        //public ErasureParameters Erasure { get; set; }
+        //public PoRParameters Por { get; set; }
+    }
+
+    public class StorageAvailability
     {
         public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral)
         {
@@ -58,44 +79,19 @@ namespace CodexPlugin
             MaxCollateral = maxCollateral;
         }
 
+        public string Id { get; set; } = string.Empty;
         public ByteSize TotalSpace { get; }
         public TimeSpan MaxDuration { get; }
         public TestToken MinPriceForTotalSpace { get; }
         public TestToken MaxCollateral { get; } 
 
-        public CodexSalesAvailabilityRequest ToApiRequest()
-        {
-            return new CodexSalesAvailabilityRequest
-            {
-                size = ToDecInt(TotalSpace.SizeInBytes),
-                duration = ToDecInt(MaxDuration.TotalSeconds),
-                maxCollateral = ToDecInt(MaxCollateral),
-                minPrice = ToDecInt(MinPriceForTotalSpace)
-            };
-        }
-
         public void Log(ILog log)
         {
             log.Log($"Making storage available... (" +
-                $"size: {TotalSpace}, " +
+                $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
                 $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +
                 $"maxCollateral: {MaxCollateral})");
         }
     }
-
-    public abstract class MarketplaceType
-    {
-        protected string ToDecInt(double d)
-        {
-            var i = new BigInteger(d);
-            return i.ToString("D");
-        }
-
-        protected string ToDecInt(TestToken t)
-        {
-            var i = new BigInteger(t.Amount);
-            return i.ToString("D");
-        }
-    }
 }

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -0,0 +1,146 @@
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace CodexPlugin
+{
+    public interface IStoragePurchaseContract
+    {
+        string PurchaseId { get; }
+        StoragePurchaseRequest Purchase { get; }
+        ContentId ContentId { get; }
+        void WaitForStorageContractSubmitted();
+        void WaitForStorageContractStarted();
+        void WaitForStorageContractFinished();
+    }
+
+    public class StoragePurchaseContract : IStoragePurchaseContract
+    {
+        private readonly ILog log;
+        private readonly CodexAccess codexAccess;
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly DateTime contractPendingUtc = DateTime.UtcNow;
+        private DateTime? contractSubmittedUtc = DateTime.UtcNow;
+        private DateTime? contractStartedUtc;
+        private DateTime? contractFinishedUtc;
+
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            Purchase = purchase;
+
+            ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
+        }
+
+        public string PurchaseId { get; }
+        public StoragePurchaseRequest Purchase { get; }
+        public ContentId ContentId { get; }
+
+        public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
+        public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
+        public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
+
+        public void WaitForStorageContractSubmitted()
+        {
+            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
+            contractSubmittedUtc = DateTime.UtcNow;
+            LogSubmittedDuration();
+            AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
+        }
+
+        public void WaitForStorageContractStarted()
+        {
+            var timeout = Purchase.Expiry + gracePeriod;
+
+            WaitForStorageContractState(timeout, "started");
+            contractStartedUtc = DateTime.UtcNow;
+            LogStartedDuration();
+            AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+            contractFinishedUtc = DateTime.UtcNow;
+            LogFinishedDuration();
+            AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+        }
+
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+
+            Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.State != lastState)
+                {
+                    lastState = purchaseStatus.State;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(sleep);
+
+                if (lastState == "errored")
+                {
+                    FrameworkAssert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
+                }
+            }
+        }
+
+        private void LogSubmittedDuration()
+        {
+            Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
+                $"( < {Time.FormatDuration(gracePeriod)})");
+        }
+
+        private void LogStartedDuration()
+        {
+            Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " +
+                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
+        }
+
+        private void LogFinishedDuration()
+        {
+            Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " +
+                $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})");
+        }
+
+        private void AssertDuration(TimeSpan? span, TimeSpan max, string message)
+        {
+            if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)");
+            if (span.Value.TotalDays >= max.TotalSeconds)
+            {
+                throw new Exception(nameof(MarketplaceAccess) +
+                    $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " +
+                    message);
+            }
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"[{PurchaseId}] {msg}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -0,0 +1,773 @@
+openapi: 3.0.3
+
+info:
+  version: 0.0.1
+  title: Codex API
+  description: "List of endpoints and interfaces available to Codex API users"
+
+security:
+  - { }
+
+components:
+  schemas:
+    MultiAddress:
+      type: string
+      description: Address of node as specified by the multi-address specification https://multiformats.io/multiaddr/
+      example: /ip4/127.0.0.1/tcp/8080
+
+    PeerId:
+      type: string
+      description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    Id:
+      type: string
+      description: 32bits identifier encoded in hex-decimal string.
+      example: 0x...
+
+    BigInt:
+      type: string
+      description: Integer represented as decimal string
+
+    Cid:
+      type: string
+      description: Content Identifier as specified at https://github.com/multiformats/cid
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    SlotId:
+      type: string
+      description: Keccak hash of the abi encoded tuple (RequestId, slot index)
+      example: 268a781e0db3f7cf36b18e5f4fdb7f586ec9edd08e5500b17c0e518a769f114a
+
+    LogLevel:
+      type: string
+      description: "One of the log levels: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL"
+      example: DEBUG
+
+    EthereumAddress:
+      type: string
+      description: Address of Ethereum address
+
+    Reward:
+      type: string
+      description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+
+    Duration:
+      type: string
+      description: The duration of the request in seconds as decimal string
+
+    ProofProbability:
+      type: string
+      description: How often storage proofs are required as decimal string
+
+    Expiry:
+      type: string
+      description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
+      default: 10 minutes
+
+    SPR:
+      type: string
+      description: Signed Peer Record (libp2p)
+
+    SPRRead:
+      type: object
+      properties:
+        spr:
+          $ref: "#/components/schemas/SPR"
+
+    PeerIdRead:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/PeerId"
+
+    ErasureParameters:
+      type: object
+      properties:
+        totalChunks:
+          type: integer
+
+    PoRParameters:
+      description: Parameters for Proof of Retrievability
+      type: object
+      properties:
+        u:
+          type: string
+        publicKey:
+          type: string
+        name:
+          type: string
+
+    Content:
+      type: object
+      description: Parameters specifying the content
+      properties:
+        cid:
+          $ref: "#/components/schemas/Cid"
+        erasure:
+          $ref: "#/components/schemas/ErasureParameters"
+        por:
+          $ref: "#/components/schemas/PoRParameters"
+
+    DebugInfo:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/PeerId"
+        addrs:
+          type: array
+          items:
+            $ref: "#/components/schemas/MultiAddress"
+        repo:
+          type: string
+          description: Path of the data repository where all nodes data are stored
+        spr:
+          $ref: "#/components/schemas/SPR"
+
+    SalesAvailability:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/Id"
+        totalSize:
+          type: string
+          description: Total size of availability's storage in bytes as decimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        minPrice:
+          type: string
+          description: Minimum price to be paid (in amount of tokens) as decimal string
+        maxCollateral:
+          type: string
+          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
+
+    SalesAvailabilityREAD:
+      allOf:
+        - $ref: "#/components/schemas/SalesAvailability"
+        - type: object
+          properties:
+            freeSize:
+              type: string
+              description: Unused size of availability's storage in bytes as decimal string
+
+    SalesAvailabilityCREATE:
+      allOf:
+        - $ref: "#/components/schemas/SalesAvailability"
+        - required:
+            - totalSize
+            - minPrice
+            - maxCollateral
+            - duration
+
+    Slot:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        slotIndex:
+          type: string
+          description: Slot Index as hexadecimal string
+
+    Reservation:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/Id"
+        availabilityId:
+          $ref: "#/components/schemas/Id"
+        size:
+          $ref: "#/components/schemas/BigInt"
+        requestId:
+          $ref: "#/components/schemas/Id"
+        slotIndex:
+          type: string
+          description: Slot Index as hexadecimal string
+
+    StorageRequestCreation:
+      type: object
+      required:
+        - reward
+        - duration
+        - proofProbability
+        - collateral
+        - expiry
+      properties:
+        duration:
+          $ref: "#/components/schemas/Duration"
+        reward:
+          $ref: "#/components/schemas/Reward"
+        proofProbability:
+          $ref: "#/components/schemas/ProofProbability"
+        nodes:
+          description: Minimal number of nodes the content should be stored on
+          type: integer
+          default: 1
+        tolerance:
+          description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
+          type: integer
+          default: 0
+        collateral:
+          type: string
+          description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
+        expiry:
+          type: string
+          description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
+    StorageAsk:
+      type: object
+      required:
+        - reward
+      properties:
+        slots:
+          description: Number of slots (eq. hosts) that the Request want to have the content spread over
+          type: integer
+        slotSize:
+          type: string
+          description: Amount of storage per slot (in bytes) as decimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        proofProbability:
+          $ref: "#/components/schemas/ProofProbability"
+        reward:
+          $ref: "#/components/schemas/Reward"
+        maxSlotLoss:
+          type: integer
+          description: Max slots that can be lost without data considered to be lost
+
+    StorageRequest:
+      type: object
+      properties:
+        id:
+          type: string
+          description: Request ID
+        client:
+          $ref: "#/components/schemas/EthereumAddress"
+        ask:
+          $ref: "#/components/schemas/StorageAsk"
+        content:
+          $ref: "#/components/schemas/Content"
+        expiry:
+          $ref: "#/components/schemas/Expiry"
+        nonce:
+          type: string
+          description: Random data
+
+    Purchase:
+      type: object
+      properties:
+        state:
+          type: string
+          description: Description of the Request's state
+        error:
+          type: string
+          description: If Request failed, then here is presented the error message
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+
+    DataList:
+      type: object
+      properties:
+        content:
+          type: array
+          items:
+            $ref: "#/components/schemas/DataItem"
+
+    DataItem:
+      type: object
+      properties:
+        cid:
+          $ref: "#/components/schemas/Cid"
+        manifest:
+          $ref: "#/components/schemas/ManifestItem"
+
+    ManifestItem:
+      type: object
+      properties:
+        rootHash:
+          $ref: "#/components/schemas/Cid"
+          description: "Root hash of the content"
+        originalBytes:
+          type: integer
+          format: int64
+          description: "Length of original content in bytes"
+        blockSize:
+          type: integer
+          description: "Size of blocks"
+        protected:
+          type: boolean
+          description: "Indicates if content is protected by erasure-coding"
+
+    Space:
+      type: object
+      properties:
+        totalBlocks:
+          description: "Number of blocks stored by the node"
+          type: integer
+          format: int64
+        quotaMaxBytes:
+          type: integer
+          format: int64
+          description: "Maximum storage space used by the node"
+        quotaUsedBytes:
+          type: integer
+          format: int64
+          description: "Amount of storage space currently in use"
+        quotaReservedBytes:
+          type: integer
+          format: int64
+          description: "Amount of storage space reserved"
+
+servers:
+  - url: "http://localhost:8080/api/codex/v1"
+
+tags:
+  - name: Marketplace
+    description: Marketplace information and operations
+  - name: Data
+    description: Data operations
+  - name: Node
+    description: Node management
+  - name: Debug
+    description: Debugging configuration
+
+paths:
+  "/connect/{peerId}":
+    get:
+      summary: "Connect to a peer"
+      description: |
+        If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
+        to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
+      tags: [ Node ]
+      operationId: connectPeer
+      parameters:
+        - in: path
+          name: peerId
+          required: true
+          schema:
+              $ref: "#/components/schemas/PeerId"
+          description: Peer that should be dialed.
+        - in: query
+          name: addrs
+          schema:
+            type: array
+            nullable: true
+            items:
+              $ref: "#/components/schemas/MultiAddress"
+          description: |
+            If supplied, it will be used to dial the peer.
+            The address has to target the listening address of the peer,
+            which is specified with the `--listen-addrs` CLI flag.
+
+      responses:
+        "200":
+          description: Successfully connected to peer
+        "400":
+          description: Peer either not found or was not possible to dial
+
+  "/data":
+    get:
+      summary: "Lists manifest CIDs stored locally in node."
+      tags: [ Data ]
+      operationId: listData
+      responses:
+        "200":
+          description: Retrieved list of content CIDs
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataList"
+
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is not found
+        "500":
+          description: Well it was bad-bad
+    post:
+      summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
+      tags: [ Data ]
+      operationId: upload
+      requestBody:
+        content:
+          application/octet-stream:
+            schema:
+              type: string
+              format: binary
+      responses:
+        "200":
+          description: CID of uploaded file
+          content:
+            text/plain:
+              schema:
+                type: string
+        "500":
+          description: Well it was bad-bad and the upload did not work out
+
+  "/data/{cid}":
+    get:
+      summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned."
+      tags: [ Data ]
+      operationId: downloadLocal
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: File to be downloaded.
+
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is unavailable locally
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network":
+    get:
+      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is not found
+        "500":
+          description: Well it was bad-bad
+
+  "/space":
+    get:
+      summary: "Gets a summary of the storage space allocation of the node."
+      tags: [ Data ]
+      operationId: space
+      responses:
+        "200":
+          description: "Summary of storage allocation"
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Space"
+
+        "500":
+          description: "It's not working as planned"
+
+  "/sales/slots":
+    get:
+      summary: "Returns active slots"
+      tags: [ Marketplace ]
+      operationId: getActiveSlots
+      responses:
+        "200":
+          description: Retrieved active slots
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Slot"
+
+        "503":
+          description: Sales are unavailable
+
+  "/sales/slots/{slotId}":
+    get:
+      summary: "Returns active slot with id {slotId} for the host"
+      tags: [ Marketplace ]
+      operationId: getActiveSlotById
+      parameters:
+        - in: path
+          name: slotId
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: File to be downloaded.
+      responses:
+        "200":
+          description: Retrieved active slot
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Slot"
+
+        "400":
+          description: Invalid or missing SlotId
+
+        "404":
+          description: Host is not in an active sale for the slot
+
+        "503":
+          description: Sales are unavailable
+
+  "/sales/availability":
+    get:
+      summary: "Returns storage that is for sale"
+      tags: [ Marketplace ]
+      operationId: getOfferedStorage
+      responses:
+        "200":
+          description: Retrieved storage availabilities of the node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/SalesAvailability"
+        "500":
+          description: Error getting unused availabilities
+        "503":
+          description: Sales are unavailable
+
+    post:
+      summary: "Offers storage for sale"
+      operationId: offerStorage
+      tags: [ Marketplace ]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SalesAvailabilityCREATE"
+      responses:
+        "201":
+          description: Created storage availability
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SalesAvailabilityREAD"
+        "400":
+          description: Invalid data input
+        "422":
+          description: Not enough node's storage quota available
+        "500":
+          description: Error reserving availability
+        "503":
+          description: Sales are unavailable
+  "/sales/availability/{id}":
+    patch:
+      summary: "Updates availability"
+      description: |
+        The new parameters will be only considered for new requests.
+        Existing Requests linked to this Availability will continue as is.
+      operationId: updateOfferedStorage
+      tags: [ Marketplace ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: ID of Availability
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SalesAvailability"
+      responses:
+        "204":
+          description: Availability successfully updated
+        "400":
+          description: Invalid data input
+        "404":
+          description: Availability not found
+        "422":
+          description: Not enough node's storage quota available
+        "500":
+          description: Error reserving availability
+        "503":
+          description: Sales are unavailable
+
+  "/sales/availability/{id}/reservations":
+    patch:
+      summary: "Get availability's reservations"
+      description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
+      operationId: getReservations
+      tags: [ Marketplace ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: ID of Availability
+      responses:
+        "200":
+          description: Retrieved storage availabilities of the node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Reservation"
+        "400":
+          description: Invalid Availability ID
+        "404":
+          description: Availability not found
+        "500":
+          description: Error getting reservations
+        "503":
+          description: Sales are unavailable
+
+  "/storage/request/{cid}":
+    post:
+      summary: "Creates a new Request for storage"
+      tags: [ Marketplace ]
+      operationId: createStorageRequest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: CID of the uploaded data that should be stored
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/StorageRequestCreation"
+      responses:
+        "200":
+          description: Returns the Request ID as decimal string
+          content:
+            text/plain:
+              schema:
+                type: string
+        "400":
+          description: Invalid or missing Request ID
+        "404":
+          description: Request ID not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/storage/purchases":
+    get:
+      summary: "Returns list of purchase IDs"
+      tags: [ Marketplace ]
+      operationId: getPurchases
+      responses:
+        "200":
+          description: Gets all purchase IDs stored in node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "503":
+          description: Purchasing is unavailable
+
+  "/storage/purchases/{id}":
+    get:
+      summary: "Returns purchase details"
+      tags: [ Marketplace ]
+      operationId: getPurchase
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: Hexadecimal ID of a Purchase
+      responses:
+        "200":
+          description: Purchase details
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Purchase"
+        "400":
+          description: Invalid or missing Purchase ID
+        "404":
+          description: Purchase not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/node/spr":
+    get:
+      summary: "Get Node's SPR"
+      operationId: getSPR
+      tags: [ Node ]
+      responses:
+        "200":
+          description: Node's SPR
+          content:
+            plain/text:
+              schema:
+                $ref: "#/components/schemas/SPR"
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SPRRead"
+        "503":
+          description: Node SPR not ready, try again later
+
+  "/node/peerid":
+    get:
+      summary: "Get Node's PeerID"
+      operationId: getPeerId
+      tags: [ Node ]
+      responses:
+        "200":
+          description: Node's Peer ID
+          content:
+            plain/text:
+              schema:
+                $ref: "#/components/schemas/PeerId"
+            application/json:
+              schema:
+                $ref: "#/components/schemas/PeerIdRead"
+
+  "/debug/chronicles/loglevel":
+    post:
+      summary: "Set log level at run time"
+      tags: [ Debug ]
+      operationId: setDebugLogLevel
+
+      parameters:
+        - in: query
+          name: level
+          required: true
+          schema:
+            $ref: "#/components/schemas/LogLevel"
+
+      responses:
+        "200":
+          description: Successfully log level set
+        "400":
+          description: Invalid or missing log level
+        "500":
+          description: Well it was bad-bad
+
+  "/debug/info":
+    get:
+      summary: "Gets node information"
+      operationId: getDebugInfo
+      tags: [ Debug ]
+      responses:
+        "200":
+          description: Node's information
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -0,0 +1,65 @@
+using System.Security.Cryptography;
+using System.Text;
+
+public static class Program
+{
+    private const string OpenApiFile = "../CodexPlugin/openapi.yaml";
+    private const string ClientFile = "../CodexPlugin/obj/openapiClient.cs";
+    private const string Search = "<INSERT-OPENAPI-YAML-HASH>";
+    private const string TargetFile = "ApiChecker.cs";
+
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("Injecting hash of 'openapi.yaml'...");
+
+        // Force client rebuild by deleting previous artifact.
+        File.Delete(ClientFile);
+
+        var hash = CreateHash();
+        // This hash is used to verify that the Codex docker image being used is compatible
+        // with the openapi.yaml being used by the Codex plugin.
+        // If the openapi.yaml files don't match, an exception is thrown.
+
+        SearchAndInject(hash);
+
+        // This program runs as the pre-build trigger for "CodexPlugin".
+        // You might be wondering why this work isn't done by a shell script.
+        // This is because this project is being run on many different platforms.
+        // (Mac, Unix, Win, but also desktop/cloud containers.)
+        // In order to not go insane trying to make a shell script that works in all possible cases,
+        // instead we use the one tool that's definitely installed in all platforms and locations
+        // when you're trying to run this plugin.
+
+        Console.WriteLine("Done!");
+    }
+
+    private static string CreateHash()
+    {
+        var file = File.ReadAllText(OpenApiFile);
+        var fileBytes = Encoding.ASCII.GetBytes(file
+            .Replace(Environment.NewLine, ""));
+
+        var sha = SHA256.Create();
+        var hash = sha.ComputeHash(fileBytes);
+        return BitConverter.ToString(hash);
+    }
+
+    private static void SearchAndInject(string hash)
+    {
+        var lines = File.ReadAllLines(TargetFile);
+        Inject(lines, hash);
+        File.WriteAllLines(TargetFile, lines);
+    }
+
+    private static void Inject(string[] lines, string hash)
+    {
+        for (var i = 0; i < lines.Length; i++)
+        {
+            if (lines[i].Contains(Search))
+            {
+                lines[i + 1] = $"        private const string OpenApiYamlHash = \"{hash}\";";
+                return;
+            }
+        }
+    }
+}

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/EthAccount.cs

@@ -0,0 +1,33 @@
+using Nethereum.Hex.HexConvertors.Extensions;
+using Nethereum.Web3.Accounts;
+
+namespace GethPlugin
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public static EthAccount GenerateNew()
+        {
+            var ecKey = Nethereum.Signer.EthECKey.GenerateKey();
+            var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex();
+            var account = new Account(privateKey);
+            var ethAddress = new EthAddress(account.Address);
+
+            return new EthAccount(ethAddress, account.PrivateKey);
+        }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
+    }
+}

ProjectPlugins/GethPlugin/EthAddress.cs

@@ -5,6 +5,7 @@
         EthAddress EthAddress { get; }
     }
 
+    [Serializable]
     public class EthAddress
     {
         public EthAddress(string address)

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -7,9 +7,9 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
+        public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
-            Containers = containers;
+            Pod = pod;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
@@ -17,9 +17,9 @@ namespace GethPlugin
             PubKey = pubKey;
         }
 
-        public RunningContainers Containers { get; }
+        public RunningPod Pod { get; }
         [JsonIgnore]
-        public RunningContainer Container {  get { return Containers.Containers.Single(); } }
+        public RunningContainer Container {  get { return Pod.Containers.Single(); } }
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -5,6 +5,7 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
+using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -24,8 +25,10 @@ namespace GethPlugin
         decimal? GetSyncedBlockNumber();
         bool IsContractAvailable(string abi, string contractAddress);
         GethBootstrapNode GetBootstrapRecord();
-        List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
+        BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
+        BlockTimeEntry GetBlockForNumber(ulong number);
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -144,14 +147,24 @@ namespace GethPlugin
             return StartInteraction().IsContractAvailable(abi, contractAddress);
         }
 
-        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new()
         {
-            return StartInteraction().GetEvents<TEvent>(address, fromBlockNumber, toBlockNumber);
+            return StartInteraction().GetEvents<TEvent>(address, blockRange);
         }
 
         public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
         {
-            return StartInteraction().GetEvents<TEvent>(address, timeRange);
+            return StartInteraction().GetEvents<TEvent>(address, ConvertTimeRangeToBlockRange(timeRange));
+        }
+
+        public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
+        {
+            return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
+        }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            return StartInteraction().GetBlockForNumber(number);
         }
 
         protected abstract NethereumInteraction StartInteraction();

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,24 +6,24 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
         {
-            return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
+            return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,15 +31,15 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
             return starter.CollectMetricsFor(scrapeTargets);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
         {
-            runningContainer = SerializeGate.Gate(runningContainer);
-            return starter.CreateAccessForTarget(runningContainer, target);
+            runningPod = SerializeGate.Gate(runningPod);
+            return starter.CreateAccessForTarget(runningPod, target);
         }
 
         public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -1,4 +1,5 @@
 using Core;
+using IdentityModel;
 using KubernetesWorkflow.Types;
 using Logging;
 using System.Globalization;
@@ -7,14 +8,16 @@ namespace MetricsPlugin
 {
     public class MetricsQuery
     {
-        private readonly IHttp http;
+        private readonly IEndpoint endpoint;
         private readonly ILog log;
 
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            http = tools.CreateHttp(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1");
+            endpoint = tools
+                .CreateHttp()
+                .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "/api/v1/");
         }
 
         public RunningContainer RunningContainer { get; }
@@ -51,7 +54,7 @@ namespace MetricsPlugin
 
         public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
             var result = MapResponseToMetrics(response);
             Log(target, result);
@@ -60,14 +63,14 @@ namespace MetricsPlugin
 
         private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
             if (response.status != "success") return null;
             return response;
         }
 
         private PrometheusQueryResponse? GetAll(string metricName)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
             return response;
         }
@@ -175,6 +178,41 @@ namespace MetricsPlugin
         {
             return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
         }
+
+        public string AsCsv()
+        {
+            var allTimestamps = Sets.SelectMany(s => s.Values.Select(v => v.Timestamp)).Distinct().OrderDescending().ToArray();
+
+            var lines = new List<string>();
+            MakeLine(lines, e =>
+            {
+                e.Add("Metrics");
+                foreach (var ts in allTimestamps) e.Add(ts.ToEpochTime().ToString());
+            });
+
+            foreach (var set in Sets)
+            {
+                MakeLine(lines, e =>
+                {
+                    e.Add(set.Name);
+                    foreach (var ts in allTimestamps)
+                    {
+                        var value = set.Values.SingleOrDefault(v => v.Timestamp == ts);
+                        if (value == null) e.Add(" ");
+                        else e.Add(value.Value.ToString());
+                    }
+                });
+            }
+
+            return string.Join(Environment.NewLine, lines.ToArray());
+        }
+
+        private void MakeLine(List<string> lines, Action<List<string>> values)
+        {
+            var list = new List<string>();
+            values(list);
+            lines.Add(string.Join(",", list));
+        }
     }
 
     public class MetricsSet

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,7 +16,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -25,16 +25,16 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig);
+            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
         {
-            var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
+            var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
         }
 

README.md

@@ -1,18 +1,25 @@
-# Distributed System Tests for Nim-Codex
+# Distributed System Tests
 
-Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
+This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v7.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0
 
+Currently, this project is mainly used for distributed testing of [Nim-Codex](https://github.com/codex-storage/nim-codex). However, its plugin-structure allows for other projects to be on-boarded (relatively) easily. (See 'contribute a plugin`.)
+
+## Tests/DistTestCore
+Library with generic distributed-testing functionality. Uses NUnit3. Reference this project to build unit-test style scenarios: setup, run test, teardown. The DistTestCore responds to the following env-vars:
+- `LOGPATH` = Path where log files will be written.
+- `DATAFILEPATH` = Path where (temporary) data files will be stored.
+- `ALWAYS_LOGS` = When set, DistTestCore will always download all container logs at the end of a test run. By default, logs are only downloaded on test failure.
+
 ## Tests/CodexTests and Tests/CodexLongTests
-These are test assemblies that use NUnit3 to perform tests against transient Codex nodes.
+These are test assemblies that use DistTestCore to perform tests against transient Codex nodes.
 Read more [HERE](/Tests/CodexTests/README.md)
 
-## Tests/ContinousTests
+## Tests/ContinuousTests
 A console application that runs tests in an endless loop against a persistent deployment of Codex nodes.
 Read more [HERE](/Tests/CodexContinuousTests/README.md)
 

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -148,7 +148,7 @@ namespace ContinuousTests
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
-            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
+            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
         }
 
         private void PerformCleanup(ILog log)
@@ -157,7 +157,7 @@ namespace ContinuousTests
             log.Log("Cleaning up test namespace...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
-            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
+            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
             log.Log("Cleanup finished.");
         }
     }