undo custom image

# Conflicts:
#	Tests/CodexContinuousTests/ContinuousTestRunner.cs

.github/workflows/dist-tests.yaml

@@ -0,0 +1,81 @@
+name: Dist Tests
+
+
+on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/job.yaml'
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch (master)
+        required: false
+        type: string
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner prefix (codex-dist-tests)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Runner command (dotnet test Tests)
+        required: false
+        type: string
+
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: codex-dist-tests
+  NAMESPACE: default
+  COMMAND: dotnet test Tests/CodexTests
+  JOB_MANIFEST: docker/job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.26.1
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Variables
+        run: |
+          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-reusable.yml

@@ -0,0 +1,177 @@
+name: Docker - Reusable
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -0,0 +1,22 @@
+name: Docker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - docker/Dockerfile
+      - docker/docker-entrypoint.sh
+      - .github/workflows/docker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit

.gitignore

@@ -1,3 +1,4 @@
 .vs
 obj
 bin
+.vscode

CONTRIBUTINGPLUGINS.MD

@@ -0,0 +1,86 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing plugins
+The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
+1. Create a library assembly in the project plugins folder.
+1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
+1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
+
+## Constructors & Tools
+Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
+```C#
+   public class MyPlugin : IProjectPlugin
+   {
+      public MyPlugin(IPluginTools tools)
+      {
+         ...
+      }
+
+      ...
+   }
+```
+
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
+
+## Plugin Interfaces
+The `IProjectPlugin` interface requires the implementation of two methods.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method.
+
+There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
+
+## Core Interface
+Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+```C#
+   public static class CoreInterfaceExtensions
+   {
+      public static MyPluginReturnType DoSomethingCool(this CoreInterface ci, string someArgument)
+      {
+         return Plugin(ci).SomethingCool(someArgument);
+      }
+
+      private static MyPlugin Plugin(CoreInterface ci)
+      {
+         return ci.GetPlugin<MyPlugin>();
+      }
+   }
+```
+
+While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
+
+## Deploying, Wrapping, and Starting
+When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
+1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
+
+Here's an example:
+```C#
+public static class CoreInterfaceExtensions
+   {
+      public static RunningContainers DeployMyProject(this CoreInterface ci, string someArgument)
+      {
+         // `RunningContainers` is a framework type. It contains all necessary information about a deployed container. It is serializable.
+         // Should you need to return any additional information, create a new type that contains it as well as the container information. Make sure it is serializable.
+         return Plugin(ci).DeployMyProjectContainer(someArgument); // <-- This method should use the `PluginTools.CreateWorkflow()` tool to deploy a container with a configuration that matches someArguments.
+      }
+
+      public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
+      {
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project.
+      }
+
+      public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
+      {
+         // Start is now nothing more than a convenience method, combining the previous two.
+         var rc = ci.DeployMyProject(someArgument);
+         return WrapMyProjectContainer(ci, rc);
+      }
+   }
+```
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+

CONTRIBUTINGTESTS.MD

@@ -0,0 +1,40 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing tests
+Do you want to write some tests using this distributed test setup? Great! Here's what you do.
+
+1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
+1. Checkout your branch.
+1. Create a new assembly in the `/Tests` folder. This can be an NUnit test assembly or simply a console app.
+1. Add Project references to `Core`, as well as any project plugin you'll be using.
+1. Write tests! Use existing tests for inspiration.
+
+## Tips for writing tests for Codex
+### Transient tests
+1. Add new code files to `Tests/CodexTests`
+1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`.
+1. When using `CodexDistTest`:
+   1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this.
+   1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
+1. When using `AutoBootstrapDistTest`:
+   1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
+   1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
+1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
+### Continuous tests
+1. Add new code files to `Tests/CodexContinousTests/Tests`
+1. Inherrit from `ContinuousTest`
+1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
+1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)
+1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against.
+1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes.
+
+### Tips for either type of test
+1. You can generate files of random test data by calling `GenerateTestFile(...)`.
+1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
+1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
+1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
+1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
+1. Tip: Commit often.
+
+## Don't forget
+1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes.

CodexDistTestCore/ByteSize.cs

@@ -1,57 +0,0 @@
-namespace CodexDistTestCore
-{
-    public class ByteSize
-    {
-        public ByteSize(long sizeInBytes)
-        {
-            SizeInBytes = sizeInBytes;
-        }
-
-        public long SizeInBytes { get; }
-    }
-
-    public static class IntExtensions
-    {
-        private const long Kilo = 1024;
-
-        public static ByteSize KB(this long i)
-        {
-            return new ByteSize(i * Kilo);
-        }
-
-        public static ByteSize MB(this long i)
-        {
-            return (i * Kilo).KB();
-        }
-
-        public static ByteSize GB(this long i)
-        {
-            return (i * Kilo).MB();
-        }
-
-        public static ByteSize TB(this long i)
-        {
-            return (i * Kilo).GB();
-        }
-
-        public static ByteSize KB(this int i)
-        {
-            return Convert.ToInt64(i).KB();
-        }
-
-        public static ByteSize MB(this int i)
-        {
-            return Convert.ToInt64(i).MB();
-        }
-
-        public static ByteSize GB(this int i)
-        {
-            return Convert.ToInt64(i).GB();
-        }
-
-        public static ByteSize TB(this int i)
-        {
-            return Convert.ToInt64(i).TB();
-        }
-    }
-}

CodexDistTestCore/CodexAPI.cs

@@ -1,17 +0,0 @@
-namespace CodexDistTestCore
-{
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public CodexDebugVersionResponse codex { get; set; } = new();
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-    }
-}

CodexDistTestCore/CodexDistTestCore.csproj

@@ -1,17 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
-    <RootNamespace>CodexDistTestCore</RootNamespace>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="KubernetesClient" Version="10.1.4" />
-    <PackageReference Include="nunit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
-  </ItemGroup>
-
-</Project>

CodexDistTestCore/CodexNodeContainer.cs

@@ -1,84 +0,0 @@
-namespace CodexDistTestCore
-{
-    public class CodexNodeContainer
-    {
-        public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort)
-        {
-            Name = name;
-            ServicePort = servicePort;
-            ServicePortName = servicePortName;
-            ApiPort = apiPort;
-            ContainerPortName = containerPortName;
-            DiscoveryPort = discoveryPort;
-            ListenPort = listenPort;
-            DataDir = dataDir;
-            MetricsPort = metricsPort;
-        }
-
-        public string Name { get; }
-        public int ServicePort { get; }
-        public string ServicePortName { get; }
-        public int ApiPort { get; }
-        public string ContainerPortName { get; }
-        public int DiscoveryPort { get; }
-        public int ListenPort { get; }
-        public string DataDir { get; }
-        public int MetricsPort { get; }
-    }
-
-    public class CodexGroupNumberSource
-    {
-        private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0);
-        private readonly NumberSource groupContainerNameSource = new NumberSource(1);
-        private readonly NumberSource servicePortSource = new NumberSource(30001);
-
-        public int GetNextCodexNodeGroupNumber()
-        {
-            return codexNodeGroupNumberSource.GetNextNumber();
-        }
-
-        public string GetNextServicePortName()
-        {
-            return $"node{groupContainerNameSource.GetNextNumber()}";
-        }
-
-        public int GetNextServicePort()
-        {
-            return servicePortSource.GetNextNumber();
-        }
-    }
-
-    public class CodexNodeContainerFactory
-    {
-        private readonly NumberSource containerNameSource = new NumberSource(1);
-        private readonly NumberSource codexPortSource = new NumberSource(8080);
-        private readonly CodexGroupNumberSource groupContainerFactory;
-
-        public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory)
-        {
-            this.groupContainerFactory = groupContainerFactory;
-        }
-
-        public CodexNodeContainer CreateNext(OfflineCodexNodes offline)
-        {
-            var n = containerNameSource.GetNextNumber();
-            return new CodexNodeContainer(
-                name: $"codex-node{n}",
-                servicePort: groupContainerFactory.GetNextServicePort(),
-                servicePortName: groupContainerFactory.GetNextServicePortName(),
-                apiPort: codexPortSource.GetNextNumber(),
-                containerPortName: $"api-{n}",
-                discoveryPort: codexPortSource.GetNextNumber(),
-                listenPort: codexPortSource.GetNextNumber(),
-                dataDir: $"datadir{n}",
-                metricsPort: GetMetricsPort(offline)
-            );
-        }
-
-        private int GetMetricsPort(OfflineCodexNodes offline)
-        {
-            if (offline.MetricsEnabled) return codexPortSource.GetNextNumber();
-            return 0;
-        }
-    }
-}

CodexDistTestCore/CodexNodeGroup.cs

@@ -1,111 +0,0 @@
-using CodexDistTestCore.Config;
-using k8s.Models;
-using System.Collections;
-
-namespace CodexDistTestCore
-{
-    public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
-    {
-        IOfflineCodexNodes BringOffline();
-        IOnlineCodexNode this[int index] { get; }
-    }
-
-    public class CodexNodeGroup : ICodexNodeGroup
-    {
-        private readonly TestLog log;
-        private readonly IK8sManager k8SManager;
-
-        public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
-        {
-            this.log = log;
-            OrderNumber = orderNumber;
-            Origin = origin;
-            this.k8SManager = k8SManager;
-            Nodes = nodes;
-
-            foreach (var n in nodes) n.Group = this;
-        }
-
-        public IOnlineCodexNode this[int index]
-        {
-            get
-            {
-                return Nodes[index];
-            }
-        }
-
-        public IOfflineCodexNodes BringOffline()
-        {
-            return k8SManager.BringOffline(this);
-        }
-
-        public int OrderNumber { get; }
-        public OfflineCodexNodes Origin { get; }
-        public OnlineCodexNode[] Nodes { get; }
-        public V1Deployment? Deployment { get; set; }
-        public V1Service? Service { get; set; }
-        public PodInfo? PodInfo { get; set; }
-
-        public CodexNodeContainer[] GetContainers()
-        {
-            return Nodes.Select(n => n.Container).ToArray();
-        }
-
-        public IEnumerator<IOnlineCodexNode> GetEnumerator()
-        {
-            return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
-        }
-
-        IEnumerator IEnumerable.GetEnumerator()
-        {
-            return Nodes.GetEnumerator();
-        }
-
-        public V1ObjectMeta GetServiceMetadata()
-        {
-            return new V1ObjectMeta
-            {
-                Name = "codex-test-entrypoint-" + OrderNumber,
-                NamespaceProperty = K8sCluster.K8sNamespace
-            };
-        }
-
-        public V1ObjectMeta GetDeploymentMetadata()
-        {
-            return new V1ObjectMeta
-            {
-                Name = "codex-test-node-" + OrderNumber,
-                NamespaceProperty = K8sCluster.K8sNamespace
-            };
-        }
-
-        public CodexNodeLog DownloadLog(IOnlineCodexNode node)
-        {
-            var logDownloader = new PodLogDownloader(log, k8SManager);
-            var n = (OnlineCodexNode)node;
-            return logDownloader.DownloadLog(n);
-        }
-
-        public Dictionary<string, string> GetSelector()
-        {
-            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };
-        }
-
-        public string Describe()
-        {
-            return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
-        }
-    }
-
-    public class PodInfo
-    {
-        public PodInfo(string name, string ip)
-        {
-            Name = name;
-            Ip = ip;
-        }
-
-        public string Name { get; }
-        public string Ip { get; }
-    }
-}

CodexDistTestCore/CodexNodeLog.cs

@@ -1,34 +0,0 @@
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public interface ICodexNodeLog
-    {
-        void AssertLogContains(string expectedString);
-    }
-
-    public class CodexNodeLog : ICodexNodeLog
-    {
-        private readonly LogFile logFile;
-
-        public CodexNodeLog(LogFile logFile)
-        {
-            this.logFile = logFile;
-        }
-
-        public void AssertLogContains(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString)) return;
-                line = streamReader.ReadLine();
-            }
-
-            Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
-        }
-    }
-}

CodexDistTestCore/Config/CodexDockerImage.cs

@@ -1,65 +0,0 @@
-using k8s.Models;
-
-namespace CodexDistTestCore.Config
-{
-    public class CodexDockerImage
-    {
-        public string GetImageTag()
-        {
-            return "thatbenbierens/nim-codex:sha-b204837";
-        }
-
-        public string GetExpectedImageRevision()
-        {
-            return "b20483";
-        }
-
-        public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment)
-        {
-            var formatter = new EnvFormatter();
-            formatter.Create(node, environment);
-            return formatter.Result;
-        }
-
-        private class EnvFormatter
-        {
-            public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
-
-            public void Create(OfflineCodexNodes node, CodexNodeContainer container)
-            {
-                AddVar("API_PORT", container.ApiPort.ToString());
-                AddVar("DATA_DIR", container.DataDir);
-                AddVar("DISC_PORT", container.DiscoveryPort.ToString());
-                AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}");
-
-                if (node.BootstrapNode != null)
-                {
-                    var debugInfo = node.BootstrapNode.GetDebugInfo();
-                    AddVar("BOOTSTRAP_SPR", debugInfo.spr);
-                }
-                if (node.LogLevel != null)
-                {
-                    AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant());
-                }
-                if (node.StorageQuota != null)
-                {
-                    AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!);
-                }
-                if (node.MetricsEnabled)
-                {
-                    AddVar("METRICS_ADDR", "0.0.0.0");
-                    AddVar("METRICS_PORT", container.MetricsPort.ToString());
-                }
-            }
-
-            private void AddVar(string key, string value)
-            {
-                Result.Add(new V1EnvVar
-                {
-                    Name = key,
-                    Value = value
-                });
-            }
-        }
-    }
-}

CodexDistTestCore/Config/FileManagerConfig.cs

@@ -1,7 +0,0 @@
-namespace CodexDistTestCore.Config
-{
-    public class FileManagerConfig
-    {
-        public const string Folder = "TestDataFiles";
-    }
-}

CodexDistTestCore/Config/K8sCluster.cs

@@ -1,40 +0,0 @@
-using k8s;
-
-namespace CodexDistTestCore.Config
-{
-    public class K8sCluster
-    {
-        public const string K8sNamespace = "codex-test-namespace";
-        private const string KubeConfigFile = "C:\\kube\\config";
-        private readonly Dictionary<Location, string> K8sNodeLocationMap = new Dictionary<Location, string>
-        {
-            { Location.BensLaptop, "worker01" },
-            { Location.BensOldGamingMachine, "worker02" },
-        };
-
-        private KubernetesClientConfiguration? config;
-
-        public KubernetesClientConfiguration GetK8sClientConfig()
-        {
-            if (config != null) return config;
-            //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
-            config = KubernetesClientConfiguration.BuildDefaultConfig();
-            return config;
-        }
-
-        public string GetIp()
-        {
-            var c = GetK8sClientConfig();
-
-            var host = c.Host.Replace("https://", "");
-
-            return host.Substring(0, host.IndexOf(':'));
-        }
-
-        public string GetNodeLabelForLocation(Location location)
-        {
-            if (location == Location.Unspecified) return string.Empty;
-            return K8sNodeLocationMap[location];
-        }
-    }
-}

CodexDistTestCore/Config/LogConfig.cs

@@ -1,7 +0,0 @@
-namespace CodexDistTestCore.Config
-{
-    public class LogConfig
-    {
-        public const string LogRoot = "D:/CodexTestLogs";
-    }
-}

CodexDistTestCore/DistTest.cs

@@ -1,120 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    [SetUpFixture]
-    public abstract class DistTest
-    {
-        private TestLog log = null!;
-        private FileManager fileManager = null!;
-        private K8sManager k8sManager = null!;
-
-        [OneTimeSetUp]
-        public void GlobalSetup()
-        {
-            // Previous test run may have been interrupted.
-            // Begin by cleaning everything up.
-            log = new TestLog();
-            fileManager = new FileManager(log);
-            k8sManager = new K8sManager(log, fileManager);
-
-            try
-            {
-                k8sManager.DeleteAllResources();
-                fileManager.DeleteAllTestFiles();
-            }
-            catch (Exception ex)
-            {
-                GlobalTestFailure.HasFailed = true;
-                log.Error($"Global setup cleanup failed with: {ex}");
-                throw;
-            }
-            log.Log("Global setup cleanup successful");
-        }
-
-        [SetUp]
-        public void SetUpDistTest()
-        {
-            if (GlobalTestFailure.HasFailed)
-            {
-                Assert.Inconclusive("Skip test: Previous test failed during clean up.");
-            }
-            else
-            {
-                var dockerImage = new CodexDockerImage();
-                log = new TestLog();
-                log.Log($"Using docker image '{dockerImage.GetImageTag()}'");
-
-                fileManager = new FileManager(log);
-                k8sManager = new K8sManager(log, fileManager);
-            }
-        }
-
-        [TearDown]
-        public void TearDownDistTest()
-        {
-            try
-            {
-                log.EndTest();
-                IncludeLogsAndMetricsOnTestFailure();
-                k8sManager.DeleteAllResources();
-                fileManager.DeleteAllTestFiles();
-            }
-            catch (Exception ex)
-            {
-                log.Error("Cleanup failed: " + ex.Message);
-                GlobalTestFailure.HasFailed = true;
-            }
-        }
-
-        public TestFile GenerateTestFile(ByteSize size)
-        {
-            return fileManager.GenerateTestFile(size);
-        }
-
-        public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
-        {
-            return new OfflineCodexNodes(k8sManager, numberOfNodes);
-        }
-
-        private void IncludeLogsAndMetricsOnTestFailure()
-        {
-            var result = TestContext.CurrentContext.Result;
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
-                if (IsDownloadingLogsAndMetricsEnabled())
-                {
-                    log.Log("Downloading all CodexNode logs and metrics because of test failure...");
-                    k8sManager.ForEachOnlineGroup(DownloadLogs);
-                    k8sManager.DownloadAllMetrics();
-                }
-                else
-                {
-                    log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
-                }
-            }
-        }
-
-        private void DownloadLogs(CodexNodeGroup group)
-        {
-            foreach (var node in group)
-            {
-                var downloader = new PodLogDownloader(log, k8sManager);
-                var n = (OnlineCodexNode)node;
-                downloader.DownloadLog(n);
-            }
-        }
-
-        private bool IsDownloadingLogsAndMetricsEnabled()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
-        }
-    }
-
-    public static class GlobalTestFailure
-    {
-        public static bool HasFailed { get; set; } = false;
-    }
-}

CodexDistTestCore/FileManager.cs

@@ -1,110 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public interface IFileManager
-    {
-        TestFile CreateEmptyTestFile();
-        TestFile GenerateTestFile(ByteSize size);
-        void DeleteAllTestFiles();
-    }
-
-    public class FileManager : IFileManager
-    {
-        public const int ChunkSize = 1024 * 1024;
-        private readonly Random random = new Random();
-        private readonly List<TestFile> activeFiles = new List<TestFile>();
-        private readonly TestLog log;
-
-        public FileManager(TestLog log)
-        {
-            if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder);
-            this.log = log;
-        }
-
-        public TestFile CreateEmptyTestFile()
-        {
-            var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin"));
-            File.Create(result.Filename).Close();
-            activeFiles.Add(result);
-            return result;
-        }
-
-        public TestFile GenerateTestFile(ByteSize size)
-        {
-            var result = CreateEmptyTestFile();
-            GenerateFileBytes(result, size);
-            log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
-            return result;
-        }
-
-        public void DeleteAllTestFiles()
-        {
-            foreach (var file in activeFiles) File.Delete(file.Filename);
-            activeFiles.Clear();
-        }
-
-        private void GenerateFileBytes(TestFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            while (bytesLeft > 0)
-            {
-                var length = Math.Min(bytesLeft, ChunkSize);
-                AppendRandomBytesToFile(result, length);
-                bytesLeft -= length;
-            }
-        }
-
-        private void AppendRandomBytesToFile(TestFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-    }
-
-    public class TestFile
-    {
-        public TestFile(string filename)
-        {
-            Filename = filename;
-        }
-
-        public string Filename { get; }
-
-        public long GetFileSize()
-        {
-            var info = new FileInfo(Filename);
-            return info.Length;
-        }
-
-        public void AssertIsEqual(TestFile? actual)
-        {
-            if (actual == null) Assert.Fail("TestFile is null.");
-            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
-
-            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
-
-            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
-            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
-
-            var bytesExpected = new byte[FileManager.ChunkSize];
-            var bytesActual = new byte[FileManager.ChunkSize];
-
-            var readExpected = 0;
-            var readActual = 0;
-
-            while (true)
-            {
-                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
-                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
-
-                if (readExpected == 0 && readActual == 0) return;
-                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
-                CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
-            }
-        }
-    }
-}

CodexDistTestCore/Http.cs

@@ -1,100 +0,0 @@
-using Newtonsoft.Json;
-using NUnit.Framework;
-using System.Net.Http.Headers;
-
-namespace CodexDistTestCore
-{
-    public class Http
-    {
-        private readonly string ip;
-        private readonly int port;
-        private readonly string baseUrl;
-
-        public Http(string ip, int port, string baseUrl)
-        {
-            this.ip = ip;
-            this.port = port;
-            this.baseUrl = baseUrl;
-
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
-        }
-
-        public string HttpGetString(string route)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                var result = Utils.Wait(client.GetAsync(url));
-                return Utils.Wait(result.Content.ReadAsStringAsync());
-            });
-        }
-
-        public T HttpGetJson<T>(string route)
-        {
-            return JsonConvert.DeserializeObject<T>(HttpGetString(route))!;
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Utils.Wait(client.PostAsync(url, content));
-
-                return Utils.Wait(response.Content.ReadAsStringAsync());
-            });
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return Retry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-
-                return Utils.Wait(client.GetStreamAsync(url));
-            });
-        }
-
-        private string GetUrl()
-        {
-            return $"http://{ip}:{port}{baseUrl}";
-        }
-
-        private static T Retry<T>(Func<T> operation)
-        {
-            var retryCounter = 0;
-
-            while (true)
-            {
-                try
-                {
-                    return operation();
-                }
-                catch (Exception exception)
-                {
-                    Timing.HttpCallRetryDelay();
-                    retryCounter++;
-                    if (retryCounter > Timing.HttpCallRetryCount())
-                    {
-                        Assert.Fail(exception.Message);
-                        throw;
-                    }
-                }
-            }
-        }
-
-        private static HttpClient GetClient()
-        {
-            var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
-            return client;
-        }
-    }
-}

CodexDistTestCore/K8sManager.cs

@@ -1,120 +0,0 @@
-namespace CodexDistTestCore
-{
-    public interface IK8sManager
-    {
-        ICodexNodeGroup BringOnline(OfflineCodexNodes node);
-        IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
-        void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
-    }
-
-    public class K8sManager : IK8sManager
-    {
-        private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
-        private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
-        private readonly KnownK8sPods knownPods = new KnownK8sPods();
-        private readonly TestLog log;
-        private readonly IFileManager fileManager;
-        private readonly MetricsAggregator metricsAggregator;
-
-        public K8sManager(TestLog log, IFileManager fileManager)
-        {
-            this.log = log;
-            this.fileManager = fileManager;
-            metricsAggregator = new MetricsAggregator(log, this);
-        }
-
-        public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
-        {
-            var online = CreateOnlineCodexNodes(offline);
-
-            K8s(k => k.BringOnline(online, offline));
-
-            log.Log($"{online.Describe()} online.");
-
-            if (offline.MetricsEnabled)
-            {
-                BringOnlineMetrics(online);
-            }
-
-            return online;
-        }
-
-        public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
-        {
-            var online = GetAndRemoveActiveNodeFor(node);
-
-            K8s(k => k.BringOffline(online));
-
-            log.Log($"{online.Describe()} offline.");
-
-            return online.Origin;
-        }
-
-        public void DeleteAllResources()
-        {
-            K8s(k => k.DeleteAllResources());
-        }
-
-        public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
-        {
-            foreach (var group in onlineCodexNodeGroups) action(group);
-        }
-
-        public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
-        {
-            K8s(k => k.FetchPodLog(node, logHandler));
-        }
-
-        public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
-        {
-            var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
-
-            PrometheusInfo? info = null;
-            K8s(k => info = k.BringOnlinePrometheus(spec));
-            return info!;
-        }
-
-        public void DownloadAllMetrics()
-        {
-            metricsAggregator.DownloadAllMetrics();
-        }
-
-        private void BringOnlineMetrics(CodexNodeGroup group)
-        {
-            var onlineNodes = group.Nodes.Cast<OnlineCodexNode>().ToArray();
-
-            metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
-        }
-
-        private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
-        {
-            var containers = CreateContainers(offline);
-            var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
-            var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
-            onlineCodexNodeGroups.Add(result);
-            return result;
-        }
-
-        private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline)
-        {
-            var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
-            var containers = new List<CodexNodeContainer>();
-            for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline));
-            return containers.ToArray();
-        }
-
-        private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
-        {
-            var n = (CodexNodeGroup)node;
-            onlineCodexNodeGroups.Remove(n);
-            return n;
-        }
-
-        private void K8s(Action<K8sOperations> action)
-        {
-            var k8s = new K8sOperations(knownPods);
-            action(k8s);
-            k8s.Close();
-        }
-    }
-}

CodexDistTestCore/K8sOperations.cs

@@ -1,324 +0,0 @@
-using CodexDistTestCore.Config;
-using k8s;
-using k8s.KubeConfigModels;
-using k8s.Models;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public class K8sOperations
-    {
-        private readonly CodexDockerImage dockerImage = new CodexDockerImage();
-        private readonly K8sCluster k8sCluster = new K8sCluster();
-        private readonly Kubernetes client;
-        private readonly KnownK8sPods knownPods;
-
-        public K8sOperations(KnownK8sPods knownPods)
-        {
-            this.knownPods = knownPods;
-
-            client = new Kubernetes(k8sCluster.GetK8sClientConfig());
-        }
-
-        public void Close()
-        {
-            client.Dispose();
-        }
-
-        public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
-        {
-            EnsureTestNamespace();
-
-            CreateDeployment(online, offline);
-            CreateService(online);
-
-            WaitUntilOnline(online);
-            FetchPodInfo(online);
-        }
-
-        public void BringOffline(CodexNodeGroup online)
-        {
-            var deploymentName = online.Deployment.Name();
-            DeleteDeployment(online);
-            DeleteService(online);
-            WaitUntilOffline(deploymentName);
-        }
-
-        public void DeleteAllResources()
-        {
-            DeleteNamespace();
-
-            WaitUntilZeroPods();
-            WaitUntilNamespaceDeleted();
-        }
-
-        public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
-        {
-            var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
-            logHandler.Log(stream);
-        }
-
-        public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
-        {
-            EnsureTestNamespace();
-
-            CreatePrometheusDeployment(spec);
-            CreatePrometheusService(spec);
-            WaitUntilPrometheusOnline(spec);
-
-            return new PrometheusInfo(spec.ServicePort, FetchNewPod());
-        }
-
-        private void FetchPodInfo(CodexNodeGroup online)
-        {
-            online.PodInfo = FetchNewPod();
-        }
-
-        private PodInfo FetchNewPod()
-        {
-            var pods = client.ListNamespacedPod(K8sNamespace).Items;
-
-            var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
-            Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
-
-            var newPod = newPods.Single();
-            var info = new PodInfo(newPod.Name(), newPod.Status.PodIP);
-
-            Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure.");
-            Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure.");
-
-            knownPods.Add(newPod.Name());
-            return info;
-        }
-
-        #region Waiting
-
-        private void WaitUntilOnline(CodexNodeGroup online)
-        {
-            WaitUntil(() =>
-            {
-                online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
-                return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
-            });
-        }
-
-        private void WaitUntilOffline(string deploymentName)
-        {
-            WaitUntil(() =>
-            {
-                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
-                return deployment == null || deployment.Status.AvailableReplicas == 0;
-            });
-        }
-
-        private void WaitUntilZeroPods()
-        {
-            WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
-        }
-
-        private void WaitUntilNamespaceDeleted()
-        {
-            WaitUntil(() => !IsTestNamespaceOnline());
-        }
-
-        private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec)
-        {
-            var deploymentName = spec.GetDeploymentName();
-            WaitUntil(() =>
-            {
-                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
-                return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            });
-        }
-
-        private void WaitUntil(Func<bool> predicate)
-        {
-            var start = DateTime.UtcNow;
-            var state = predicate();
-            while (!state)
-            {
-                if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
-                {
-                    Assert.Fail("K8s operation timed out.");
-                    throw new TimeoutException();
-                }
-
-                Timing.WaitForK8sServiceDelay();
-                state = predicate();
-            }
-        }
-
-        #endregion
-
-        #region Service management
-
-        private void CreateService(CodexNodeGroup online)
-        {
-            var serviceSpec = new V1Service
-            {
-                ApiVersion = "v1",
-                Metadata = online.GetServiceMetadata(),
-                Spec = new V1ServiceSpec
-                {
-                    Type = "NodePort",
-                    Selector = online.GetSelector(),
-                    Ports = CreateServicePorts(online)
-                }
-            };
-
-            online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
-        }
-
-        private List<V1ServicePort> CreateServicePorts(CodexNodeGroup online)
-        {
-            var result = new List<V1ServicePort>();
-            var containers = online.GetContainers();
-            foreach (var container in containers)
-            {
-                result.Add(new V1ServicePort
-                {
-                    Name = container.ServicePortName,
-                    Protocol = "TCP",
-                    Port = container.ApiPort,
-                    TargetPort = container.ContainerPortName,
-                    NodePort = container.ServicePort
-                });
-            }
-            return result;
-        }
-
-        private void DeleteService(CodexNodeGroup online)
-        {
-            if (online.Service == null) return;
-            client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
-            online.Service = null;
-        }
-
-        private void CreatePrometheusService(K8sPrometheusSpecs spec)
-        {
-            client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
-        }
-
-        #endregion
-
-        #region Deployment management
-
-        private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
-        {
-            var deploymentSpec = new V1Deployment
-            {
-                ApiVersion = "apps/v1",
-                Metadata = online.GetDeploymentMetadata(),
-                Spec = new V1DeploymentSpec
-                {
-                    Replicas = 1,
-                    Selector = new V1LabelSelector
-                    {
-                        MatchLabels = online.GetSelector()
-                    },
-                    Template = new V1PodTemplateSpec
-                    {
-                        Metadata = new V1ObjectMeta
-                        {
-                            Labels = online.GetSelector()
-                        },
-                        Spec = new V1PodSpec
-                        {
-                            NodeSelector = CreateNodeSelector(offline),
-                            Containers = CreateDeploymentContainers(online, offline)
-                        }
-                    }
-                }
-            };
-
-            online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
-        }
-
-        private IDictionary<string, string> CreateNodeSelector(OfflineCodexNodes offline)
-        {
-            if (offline.Location == Location.Unspecified) return new Dictionary<string, string>();
-
-            return new Dictionary<string, string> 
-            { 
-                { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) }
-            };
-        }
-
-        private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
-        {
-            var result = new List<V1Container>();
-            var containers = online.GetContainers();
-            foreach (var container in containers)
-            {
-                result.Add(new V1Container
-                {
-                    Name = container.Name,
-                    Image = dockerImage.GetImageTag(),
-                    Ports = new List<V1ContainerPort>
-                    {
-                        new V1ContainerPort
-                        {
-                            ContainerPort = container.ApiPort,
-                            Name = container.ContainerPortName
-                        }
-                    },
-                    Env = dockerImage.CreateEnvironmentVariables(offline, container)
-                });
-            }
-
-            return result;
-        }
-
-        private void DeleteDeployment(CodexNodeGroup online)
-        {
-            if (online.Deployment == null) return;
-            client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
-            online.Deployment = null;
-        }
-
-        private void CreatePrometheusDeployment(K8sPrometheusSpecs spec)
-        {
-            client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace);
-        }
-
-        #endregion
-
-        #region Namespace management
-
-        private void EnsureTestNamespace()
-        {
-            if (IsTestNamespaceOnline()) return;
-
-            var namespaceSpec = new V1Namespace
-            {
-                ApiVersion = "v1",
-                Metadata = new V1ObjectMeta
-                {
-                    Name = K8sNamespace,
-                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
-                }
-            };
-            client.CreateNamespace(namespaceSpec);
-        }
-
-        private void DeleteNamespace()
-        {
-            if (IsTestNamespaceOnline())
-            {
-                client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
-            }
-        }
-
-        private string K8sNamespace
-        {
-            get { return K8sCluster.K8sNamespace; }
-        }
-
-        #endregion
-
-        private bool IsTestNamespaceOnline()
-        {
-            return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
-        }
-    }
-}

CodexDistTestCore/K8sPrometheusSpecs.cs

@@ -1,122 +0,0 @@
-using CodexDistTestCore.Config;
-using k8s.Models;
-
-namespace CodexDistTestCore
-{
-    public class K8sPrometheusSpecs
-    {
-        public const string ContainerName = "dtest-prom";
-        public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
-        private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
-        private const string portName = "prom-1";
-        private readonly string config;
-
-        public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
-        {
-            ServicePort = servicePort;
-            PrometheusNumber = prometheusNumber;
-            this.config = config;
-        }
-
-        public int ServicePort { get; }
-        public int PrometheusNumber { get; }
-
-        public string GetDeploymentName()
-        {
-            return "test-prom" + PrometheusNumber;
-        }
-
-        public V1Deployment CreatePrometheusDeployment()
-        {
-            var deploymentSpec = new V1Deployment
-            {
-                ApiVersion = "apps/v1",
-                Metadata =  new V1ObjectMeta
-                {
-                    Name = GetDeploymentName(),
-                    NamespaceProperty = K8sCluster.K8sNamespace
-                },
-                Spec = new V1DeploymentSpec
-                {
-                    Replicas = 1,
-                    Selector = new V1LabelSelector
-                    {
-                        MatchLabels = CreateSelector()
-                    },
-                    Template = new V1PodTemplateSpec
-                    {
-                        Metadata = new V1ObjectMeta
-                        {
-                            Labels = CreateSelector()
-                        },
-                        Spec = new V1PodSpec
-                        {
-                            Containers = new List<V1Container>
-                            {
-                                new V1Container
-                                {
-                                    Name = ContainerName,
-                                    Image = dockerImage,
-                                    Ports = new List<V1ContainerPort>
-                                    {
-                                        new V1ContainerPort
-                                        {
-                                            ContainerPort = 9090,
-                                            Name = portName
-                                        }
-                                    },
-                                    Env = new List<V1EnvVar>
-                                    {
-                                        new V1EnvVar
-                                        {
-                                            Name = "PROM_CONFIG",
-                                            Value = config
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            };
-
-            return deploymentSpec;
-        }
-
-        public V1Service CreatePrometheusService()
-        {
-            var serviceSpec = new V1Service
-            {
-                ApiVersion = "v1",
-                Metadata = new V1ObjectMeta
-                {
-                    Name = "codex-prom-service" + PrometheusNumber,
-                    NamespaceProperty = K8sCluster.K8sNamespace
-                },
-                Spec = new V1ServiceSpec
-                {
-                    Type = "NodePort",
-                    Selector = CreateSelector(),
-                    Ports = new List<V1ServicePort>
-                    {
-                        new V1ServicePort
-                        {
-                            Name = "prom-service" + PrometheusNumber,
-                            Protocol = "TCP",
-                            Port = 9090,
-                            TargetPort = portName,
-                            NodePort = ServicePort
-                        }
-                    }
-                }
-            };
-
-            return serviceSpec;
-        }
-
-        private Dictionary<string, string> CreateSelector()
-        {
-            return new Dictionary<string, string> { { "test-prom", "dtest-prom" } };
-        }
-    }
-}

CodexDistTestCore/MetricsAccess.cs

@@ -1,63 +0,0 @@
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-
-namespace CodexDistTestCore
-{
-    public interface IMetricsAccess
-    {
-        void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
-    }
-
-    public class MetricsUnavailable : IMetricsAccess
-    {
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-
-    public class MetricsAccess : IMetricsAccess
-    {
-        private readonly MetricsQuery query;
-        private readonly OnlineCodexNode node;
-
-        public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
-        {
-            this.query = query;
-            this.node = node;
-        }
-
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            var metricSet = GetMetricWithTimeout(metricName, node);
-            var metricValue = metricSet.Values[0].Value;
-            Assert.That(metricValue, constraint, message);
-        }
-
-        private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
-        {
-            var start = DateTime.UtcNow;
-
-            while (true)
-            {
-                var mostRecent = GetMostRecent(metricName, node);
-                if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
-                {
-                    Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
-                    throw new TimeoutException();
-                }
-
-                Utils.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
-        {
-            var result = query.GetMostRecent(metricName, node);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
-        }
-    }
-}

CodexDistTestCore/MetricsAggregator.cs

@@ -1,78 +0,0 @@
-using NUnit.Framework;
-using System.Text;
-
-namespace CodexDistTestCore
-{
-    public class MetricsAggregator
-    {
-        private readonly NumberSource prometheusNumberSource = new NumberSource(0);
-        private readonly TestLog log;
-        private readonly K8sManager k8sManager;
-        private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
-
-        public MetricsAggregator(TestLog log, K8sManager k8sManager)
-        {
-            this.log = log;
-            this.k8sManager = k8sManager;
-        }
-
-        public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
-        {
-            log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
-
-            var config = GeneratePrometheusConfig(nodes);
-            var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
-            var query = new MetricsQuery(prometheus);
-            activePrometheuses.Add(query, nodes);
-
-            log.Log("Metrics service started.");
-
-            foreach(var node in nodes)
-            {
-                node.Metrics = new MetricsAccess(query, node);
-            }
-        }
-
-        public void DownloadAllMetrics()
-        {
-            var download = new MetricsDownloader(log, activePrometheuses);
-            download.DownloadAllMetrics();
-        }
-
-        private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
-        {
-            var config = "";
-            config += "global:\n";
-            config += "  scrape_interval: 30s\n";
-            config += "  scrape_timeout: 10s\n";
-            config += "\n";
-            config += "scrape_configs:\n";
-            config += "  - job_name: services\n";
-            config += "    metrics_path: /metrics\n";
-            config += "    static_configs:\n";
-            config += "      - targets:\n";
-
-            foreach (var node in nodes)
-            {
-                var ip = node.Group.PodInfo!.Ip;
-                var port = node.Container.MetricsPort;
-                config += $"          - '{ip}:{port}'\n";
-            }
-
-            var bytes = Encoding.ASCII.GetBytes(config);
-            return Convert.ToBase64String(bytes);
-        }
-    }
-
-    public class PrometheusInfo
-    {
-        public PrometheusInfo(int servicePort, PodInfo podInfo)
-        {
-            ServicePort = servicePort;
-            PodInfo = podInfo;
-        }
-
-        public int ServicePort { get; }
-        public PodInfo PodInfo { get; }
-    }
-}

CodexDistTestCore/OfflineCodexNodes.cs

@@ -1,96 +0,0 @@
-namespace CodexDistTestCore
-{
-    public interface IOfflineCodexNodes
-    {
-        IOfflineCodexNodes At(Location location);
-        IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
-        IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
-        IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
-        IOfflineCodexNodes EnableMetrics();
-        ICodexNodeGroup BringOnline();
-    }
-
-    public enum CodexLogLevel
-    {
-        Trace,
-        Debug,
-        Info,
-        Warn,
-        Error
-    }
-
-    public enum Location
-    {
-        Unspecified,
-        BensLaptop,
-        BensOldGamingMachine,
-    }
-
-    public class OfflineCodexNodes : IOfflineCodexNodes
-    {
-        private readonly IK8sManager k8SManager;
-        
-        public int NumberOfNodes { get; }
-        public Location Location { get; private set; }
-        public CodexLogLevel? LogLevel { get; private set; }
-        public IOnlineCodexNode? BootstrapNode { get; private set; }
-        public ByteSize? StorageQuota { get; private set; }
-        public bool MetricsEnabled { get; private set; }
-
-        public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
-        {
-            this.k8SManager = k8SManager;
-            NumberOfNodes = numberOfNodes;
-            Location = Location.Unspecified;
-            MetricsEnabled = false;
-        }
-
-        public ICodexNodeGroup BringOnline()
-        {
-            return k8SManager.BringOnline(this);
-        }
-
-        public IOfflineCodexNodes At(Location location)
-        {
-            Location = location;
-            return this;
-        }
-
-        public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
-        {
-            BootstrapNode = node;
-            return this;
-        }
-
-        public IOfflineCodexNodes WithLogLevel(CodexLogLevel level)
-        {
-            LogLevel = level;
-            return this;
-        }
-
-        public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota)
-        {
-            StorageQuota = storageQuota;
-            return this;
-        }
-
-        public IOfflineCodexNodes EnableMetrics()
-        {
-            MetricsEnabled = true;
-            return this;
-        }
-
-        public string Describe()
-        {
-            var args = string.Join(',', DescribeArgs());
-            return $"{NumberOfNodes} CodexNodes with [{args}]";
-        }
-
-        private IEnumerable<string> DescribeArgs()
-        {
-            if (LogLevel != null) yield return ($"LogLevel={LogLevel}");
-            if (BootstrapNode != null) yield return ("BootstrapNode=set");
-            if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}");
-        }
-    }
-}

CodexDistTestCore/OnlineCodexNode.cs

@@ -1,137 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public interface IOnlineCodexNode
-    {
-        CodexDebugResponse GetDebugInfo();
-        ContentId UploadFile(TestFile file);
-        TestFile? DownloadContent(ContentId contentId);
-        void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
-        IMetricsAccess Metrics { get; }
-    }
-
-    public class OnlineCodexNode : IOnlineCodexNode
-    {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
-        private const string UploadFailedMessage = "Unable to store block";
-
-        private readonly K8sCluster k8sCluster = new K8sCluster();
-        private readonly TestLog log;
-        private readonly IFileManager fileManager;
-
-        public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container)
-        {
-            this.log = log;
-            this.fileManager = fileManager;
-            Container = container;
-        }
-
-        public CodexNodeContainer Container { get; }
-        public CodexNodeGroup Group { get; internal set; } = null!;
-        public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable();
-
-        public string GetName()
-        {
-            return $"<{Container.Name}>";
-        }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            var response = Http().HttpGetJson<CodexDebugResponse>("debug/info");
-            Log($"Got DebugInfo with id: '{response.id}'.");
-            return response;
-        }
-
-        public ContentId UploadFile(TestFile file)
-        {
-            Log($"Uploading file of size {file.GetFileSize()}...");
-            using var fileStream = File.OpenRead(file.Filename);
-            var response = Http().HttpPostStream("upload", fileStream);
-            if (response.StartsWith(UploadFailedMessage))
-            {
-                Assert.Fail("Node failed to store block.");
-            }
-            Log($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        public TestFile? DownloadContent(ContentId contentId)
-        {
-            Log($"Downloading for contentId: '{contentId.Id}'...");
-            var file = fileManager.CreateEmptyTestFile();
-            DownloadToFile(contentId.Id, file);
-            Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
-            return file;
-        }
-
-        public void ConnectToPeer(IOnlineCodexNode node)
-        {
-            var peer = (OnlineCodexNode)node;
-
-            Log($"Connecting to peer {peer.GetName()}...");
-            var peerInfo = node.GetDebugInfo();
-            var peerId = peerInfo.id;
-            var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo);
-
-            var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-
-            Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
-            Log($"Successfully connected to peer {peer.GetName()}.");
-        }
-
-        public ICodexNodeLog DownloadLog()
-        {
-            return Group.DownloadLog(this);
-        }
-
-        public string Describe()
-        {
-            return $"{Group.Describe()} contains {GetName()}";
-        }
-
-        private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
-        {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip);
-        }
-
-        private void DownloadToFile(string contentId, TestFile file)
-        {
-            using var fileStream = File.OpenWrite(file.Filename);
-            using var downloadStream = Http().HttpGetStream("download/" + contentId);
-            downloadStream.CopyTo(fileStream);
-        }
-
-        private Http Http()
-        {
-            return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1");
-        }
-
-        private void Log(string msg)
-        {
-            log.Log($"{GetName()}: {msg}");
-        }
-    }
-
-    public class ContentId
-    {
-        public ContentId(string id)
-        {
-            Id = id;
-        }
-
-        public string Id { get; }
-    }
-}

CodexDistTestCore/PodLogDownloader.cs

@@ -1,73 +0,0 @@
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public interface IPodLogHandler
-    {
-        void Log(Stream log);
-    }
-
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
-    {
-        public DontDownloadLogsAndMetricsOnFailureAttribute()
-            : base(Timing.UseLongTimeoutsKey)
-        {
-        }
-    }
-
-    public class PodLogDownloader
-    {
-        public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure";
-
-        private readonly TestLog log;
-        private readonly IK8sManager k8SManager;
-
-        public PodLogDownloader(TestLog log, IK8sManager k8sManager)
-        {
-            this.log = log;
-            k8SManager = k8sManager;
-        }
-
-        public CodexNodeLog DownloadLog(OnlineCodexNode node)
-        {
-            var description = node.Describe();
-            var subFile = log.CreateSubfile();
-
-            log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
-            var handler = new PodLogDownloadHandler(description, subFile);
-            k8SManager.FetchPodLog(node, handler);
-            return handler.CreateCodexNodeLog();
-        }
-    }
-
-    public class PodLogDownloadHandler : IPodLogHandler
-    {
-        private readonly string description;
-        private readonly LogFile log;
-
-        public PodLogDownloadHandler(string description, LogFile log)
-        {
-            this.description = description;
-            this.log = log;
-        }
-
-        public CodexNodeLog CreateCodexNodeLog()
-        {
-            return new CodexNodeLog(log);
-        }
-
-        public void Log(Stream stream)
-        {
-            log.Write($"{description} -->> {log.FilenameWithoutPath}");
-            log.WriteRaw(description);
-            var reader = new StreamReader(stream);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                log.WriteRaw(line);
-                line = reader.ReadLine();
-            }
-        }
-    }
-}

CodexDistTestCore/TestLog.cs

@@ -1,144 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    public class TestLog
-    {
-        private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        private readonly LogFile file;
-        private readonly DateTime now;
-
-        public TestLog()
-        {
-            now = DateTime.UtcNow;
-
-            var name = GetTestName();
-            file = new LogFile(now, name);
-
-            Log($"Begin: {name}");
-        }
-
-        public void Log(string message)
-        {
-            file.Write(message);
-        }
-
-        public void Error(string message)
-        {
-            Log($"[ERROR] {message}");
-        }
-
-        public void EndTest()
-        {
-            var result = TestContext.CurrentContext.Result;
-
-            Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
-            if (!string.IsNullOrEmpty(result.Message))
-            {
-                Log(result.Message);
-                Log($"{result.StackTrace}");
-            }
-
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
-                RenameLogFile();
-            }
-        }
-
-        private void RenameLogFile()
-        {
-            file.ConcatToFilename("_FAILED");
-        }
-
-        public LogFile CreateSubfile(string ext = "log")
-        {
-            return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
-        }
-
-        private static string GetTestName()
-        {
-            var test = TestContext.CurrentContext.Test;
-            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
-            var args = FormatArguments(test);
-            return $"{className}.{test.MethodName}{args}";
-        }
-
-        private static string FormatArguments(TestContext.TestAdapter test)
-        {
-            if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
-        }
-    }
-
-    public class LogFile
-    {
-        private readonly DateTime now;
-        private string name;
-        private readonly string ext;
-        private readonly string filepath;
-
-        public LogFile(DateTime now, string name, string ext = "log")
-        {
-            this.now = now;
-            this.name = name;
-            this.ext = ext;
-
-            filepath = Path.Join(
-                LogConfig.LogRoot,
-                $"{now.Year}-{Pad(now.Month)}",
-                Pad(now.Day));
-
-            Directory.CreateDirectory(filepath);
-
-            GenerateFilename();
-        }
-
-        public string FullFilename { get; private set; } = string.Empty;
-        public string FilenameWithoutPath { get; private set; } = string.Empty;
-
-        public void Write(string message)
-        {
-            WriteRaw($"{GetTimestamp()} {message}");
-        }
-
-        public void WriteRaw(string message)
-        {
-            try
-            {
-                File.AppendAllLines(FullFilename, new[] { message });
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine("Writing to log has failed: " + ex);
-            }
-        }
-
-        public void ConcatToFilename(string toAdd)
-        {
-            var oldFullName = FullFilename;
-
-            name += toAdd;
-
-            GenerateFilename();
-
-            File.Move(oldFullName, FullFilename);
-        }
-
-        private static string Pad(int n)
-        {
-            return n.ToString().PadLeft(2, '0');
-        }
-
-        private static string GetTimestamp()
-        {
-            return $"[{DateTime.UtcNow.ToString("u")}]";
-        }
-
-        private void GenerateFilename()
-        {
-            FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}";
-            FullFilename = Path.Combine(filepath, FilenameWithoutPath);
-        }
-    }
-}

CodexDistTestCore/Timing.cs

@@ -1,131 +0,0 @@
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class UseLongTimeoutsAttribute : PropertyAttribute
-    {
-        public UseLongTimeoutsAttribute()
-            : base(Timing.UseLongTimeoutsKey)
-        {
-        }
-    }
-
-    public static class Timing
-    {
-        public const string UseLongTimeoutsKey = "UseLongTimeouts";
-
-        public static TimeSpan HttpCallTimeout()
-        {
-            return GetTimes().HttpCallTimeout();
-        }
-
-        public static int HttpCallRetryCount()
-        {
-            return GetTimes().HttpCallRetryCount();
-        }
-
-        public static void HttpCallRetryDelay()
-        {
-            Utils.Sleep(GetTimes().HttpCallRetryDelay());
-        }
-
-        public static void WaitForK8sServiceDelay()
-        {
-            Utils.Sleep(GetTimes().WaitForK8sServiceDelay());
-        }
-
-        public static TimeSpan K8sOperationTimeout()
-        {
-            return GetTimes().K8sOperationTimeout();
-        }
-
-        public static TimeSpan WaitForMetricTimeout()
-        {
-            return GetTimes().WaitForMetricTimeout();
-        }
-
-        private static ITimeSet GetTimes()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
-    }
-
-    public interface ITimeSet
-    {
-        TimeSpan HttpCallTimeout();
-        int HttpCallRetryCount();
-        TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
-        TimeSpan K8sOperationTimeout();
-        TimeSpan WaitForMetricTimeout();
-    }
-
-    public class DefaultTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public int HttpCallRetryCount()
-        {
-            return 5;
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(3);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(1);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-    }
-
-    public class LongTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromHours(2);
-        }
-
-        public int HttpCallRetryCount()
-        {
-            return 2;
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(15);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-    }
-}

CodexDistTestCore/Utils.cs

@@ -1,16 +0,0 @@
-namespace CodexDistTestCore
-{
-    public static class Utils
-    {
-        public static void Sleep(TimeSpan span)
-        {
-            Thread.Sleep(span);
-        }
-
-        public static T Wait<T>(Task<T> task)
-        {
-            task.Wait();
-            return task.Result;
-        }
-    }
-}

Framework/ArgsUniform/ArgsUniform.cs

@@ -0,0 +1,255 @@
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class ArgsUniform<T>
+    {
+        private readonly Action printAppInfo;
+        private readonly object? defaultsProvider;
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private const int cliStart = 8;
+        private const int shortStart = 38;
+        private const int envStart = 48;
+        private const int descStart = 80;
+
+        public ArgsUniform(Action printAppInfo, params string[] args)
+            : this(printAppInfo, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
+            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
+            : this(printAppInfo, null!, env, args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
+        {
+            this.printAppInfo = printAppInfo;
+            this.defaultsProvider = defaultsProvider;
+            this.env = env;
+            this.args = args;
+        }
+
+        public T Parse(bool printResult = false)
+        {
+            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+            {
+                printAppInfo();
+                PrintHelp();
+                throw new Exception();
+            }
+
+            var result = Activator.CreateInstance<T>();
+            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            var missingRequired = new List<PropertyInfo>();
+            foreach (var uniformProperty in uniformProperties)
+            {
+                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
+                if (attr != null)
+                {
+                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    {
+                        {
+                            missingRequired.Add(uniformProperty);
+                        }
+                    }
+                }
+            }
+
+            if (missingRequired.Any())
+            {
+                PrintResults(result, uniformProperties);
+                Print("");
+                foreach (var missing in missingRequired)
+                {
+                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
+                    var exampleArg = $"--{attr.Arg}=...";
+                    var exampleEnvVar = $"{attr.EnvVar}=...";
+                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
+                }
+
+                PrintHelp();
+                throw new ArgumentException("Unable to assemble all required arguments");
+            }
+
+            if (printResult)
+            {
+                PrintResults(result, uniformProperties);
+            }
+
+            return result;
+        }
+
+        private void PrintResults(T result, PropertyInfo[] uniformProperties)
+        {
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
+        public void PrintHelp()
+        {
+            Print("");
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
+            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
+            foreach (var attr in attrs)
+            {
+                var a = attr!;
+                var optional = !a.Required ? " *" : "";
+                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+            }
+            Print("");
+        }
+
+        private void Print(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+
+        private void PrintAligned(string cli, string s, string env, string desc)
+        {
+            Console.CursorLeft = cliStart;
+            Console.Write(cli);
+            Console.CursorLeft = shortStart;
+            Console.Write(s);
+            Console.CursorLeft = envStart;
+            Console.Write(env);
+            Console.CursorLeft = descStart;
+            Console.Write(desc + Environment.NewLine);
+        }
+
+        private object GetDefaultValue(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+
+        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var currentValue = uniformProperty.GetValue(result);
+            var isEmptryString = (currentValue as string) == string.Empty;
+            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
+
+            if (defaultsProvider == null) return false;
+
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return false;
+
+            var value = defaultProperty.GetValue(defaultsProvider);
+            if (value != null)
+            {
+                return Assign(result, uniformProperty, value);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    
+                    throw new NotSupportedException();
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+    }
+}

Framework/ArgsUniform/ArgsUniform.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="I-Env" Version="1.0.0" />
+  </ItemGroup>
+
+</Project>

Framework/ArgsUniform/ExampleUser.cs

@@ -0,0 +1,34 @@
+namespace ArgsUniform
+{
+    public class ExampleUser
+    {
+        public class Args
+        {
+            [Uniform("aaa", "a", "AAA", false, "Sets the AAA!")]
+            public string Aaa { get; set; } = string.Empty;
+
+            [Uniform("bbb", "b", "BBB", true, "Sets that BBB")]
+            public string Bbb { get; set; } = string.Empty;
+        }
+
+        public class DefaultsProvider
+        {
+            public string Aaa { get { return "non-static operation"; } }
+        }
+
+        public void Example()
+        {
+            // env var: "AAA=BBB"
+            var args = "--ccc=ddd";
+
+            var uniform = new ArgsUniform<Args>(PrintHelp, new DefaultsProvider(), args);
+
+            var aaa = uniform.Parse();
+        }
+
+        private static void PrintHelp()
+        {
+            Console.WriteLine("Help text!");
+        }
+    }
+}

Framework/ArgsUniform/UniformAttribute.cs

@@ -0,0 +1,20 @@
+namespace ArgsUniform
+{
+    public class UniformAttribute : Attribute
+    {
+        public UniformAttribute(string arg, string argShort, string envVar, bool required, string description)
+        {
+            Arg = arg;
+            ArgShort = argShort;
+            EnvVar = envVar;
+            Required = required;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string ArgShort { get; }
+        public string EnvVar { get; }
+        public bool Required { get; }
+        public string Description { get; }
+    }
+}

Framework/Core/Core.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/Core/CoreInterface.cs

@@ -0,0 +1,55 @@
+using KubernetesWorkflow;
+
+namespace Core
+{
+    public sealed class CoreInterface
+    {
+        private readonly EntryPoint entryPoint;
+
+        internal CoreInterface(EntryPoint entryPoint)
+        {
+            this.entryPoint = entryPoint;
+        }
+
+        public T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return entryPoint.GetPlugin<T>();
+        }
+
+        public IKnownLocations GetKnownLocations()
+        {
+            return entryPoint.Tools.CreateWorkflow().GetAvailableLocations();
+        }
+
+        public IDownloadedLog DownloadLog(IHasContainer containerSource, int? tailLines = null)
+        {
+            return DownloadLog(containerSource.Container, tailLines);
+        }
+
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            var file = entryPoint.Tools.GetLog().CreateSubfile();
+            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
+            var logHandler = new LogDownloadHandler(container.Name, file);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
+            return logHandler.DownloadLog();
+        }
+
+        public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)
+        {
+            return ExecuteContainerCommand(containerSource.Container, command, args);
+        }
+
+        public string ExecuteContainerCommand(RunningContainer container, string command, params string[] args)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            return workflow.ExecuteCommand(container, command, args);
+        }
+    }
+
+    public interface IHasContainer
+    {
+        RunningContainer Container { get; }
+    }
+}

Framework/Core/DownloadedLog.cs

@@ -0,0 +1,62 @@
+using Logging;
+
+namespace Core
+{
+    public interface IDownloadedLog
+    {
+        bool DoesLogContain(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
+    }
+
+    internal class DownloadedLog : IDownloadedLog
+    {
+        private readonly LogFile logFile;
+
+        internal DownloadedLog(LogFile logFile)
+        {
+            this.logFile = logFile;
+        }
+
+        public bool DoesLogContain(string expectedString)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (line.Contains(expectedString)) return true;
+                line = streamReader.ReadLine();
+            }
+
+            //Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
+            return false;
+        }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
+    }
+}

Framework/Core/EntryPoint.cs

@@ -0,0 +1,52 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public class EntryPoint
+    {
+        private readonly IToolsFactory toolsFactory;
+        private readonly PluginManager manager = new PluginManager();
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+
+            Tools = toolsFactory.CreateTools();
+            manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
+        }
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
+            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+        {
+        }
+
+        public IPluginTools Tools { get; }
+
+        public void Announce()
+        {
+            manager.AnnouncePlugins();
+        }
+
+        public Dictionary<string, string> GetPluginMetadata()
+        {
+            return manager.GatherPluginMetadata().Get();
+        }
+
+        public CoreInterface CreateInterface()
+        {
+            return new CoreInterface(this);
+        }
+
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        {
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return manager.GetPlugin<T>();
+        }
+    }
+}

Framework/Core/Http.cs

@@ -0,0 +1,214 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IHttp
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        string HttpPostString(string route, string body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T TryJsonDeserialize<T>(string json);
+    }
+
+    internal class Http : IHttp
+    {
+        private readonly ILog log;
+        private readonly ITimeSet timeSet;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
+        private readonly string? logAlias;
+
+        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        {
+        }
+
+        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
+            this.logAlias = logAlias;
+            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
+            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
+        }
+
+        public string HttpGetString(string route)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, "");
+                var result = Time.Wait(client.GetAsync(url));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str; ;
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            var json = HttpGetString(route);
+            return TryJsonDeserialize<T>(json);
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            var response = PostJson(route, body);
+            var json = Time.Wait(response.Content.ReadAsStringAsync());
+            if (!response.IsSuccessStatusCode)
+            {
+                throw new HttpRequestException(json);
+            }
+            Log(GetUrl() + route, json);
+            return TryJsonDeserialize<TResponse>(json);
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            var response = PostJson(route, body);
+            return Time.Wait(response.Content.ReadAsStringAsync());
+        }
+
+        public string HttpPostString(string route, string body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            var response = HttpPostString(route, body);
+            if (response == null) throw new Exception("Received no response.");
+            var result = JsonConvert.DeserializeObject<TResponse>(response);
+            if (result == null) throw new Exception("Failed to deserialize response");
+            return result;
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return Retry(() =>
+            {
+                var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T TryJsonDeserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count() > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
+                return Time.Wait(client.PostAsync(url, content));
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+
+        private T Retry<T>(Func<T> operation, string description)
+        {
+            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+        }
+
+        private HttpClient GetClient()
+        {
+            var client = new HttpClient();
+            client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
+            return client;
+        }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
+    }
+}

Framework/Core/LogDownloadHandler.cs

@@ -0,0 +1,28 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    internal class LogDownloadHandler : LogHandler, ILogHandler
+    {
+        private readonly LogFile log;
+
+        internal LogDownloadHandler(string description, LogFile log)
+        {
+            this.log = log;
+
+            log.Write($"{description} -->> {log.FullFilename}");
+            log.WriteRaw(description);
+        }
+
+        internal IDownloadedLog DownloadLog()
+        {
+            return new DownloadedLog(log);
+        }
+
+        protected override void ProcessLine(string line)
+        {
+            log.WriteRaw(line);
+        }
+    }
+}

Framework/Core/PluginFinder.cs

@@ -0,0 +1,46 @@
+using System.Reflection;
+
+namespace Core
+{
+    internal static class PluginFinder
+    {
+        private static Type[]? pluginTypes = null;
+
+        internal static Type[] GetPluginTypes()
+        {
+            if (pluginTypes != null) return pluginTypes;
+
+            // Reflection can be costly. Do this only once.
+            FindAndLoadPluginAssemblies();
+
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t =>
+                typeof(IProjectPlugin).IsAssignableFrom(t) &&
+                !t.IsAbstract)
+            ).ToArray();
+
+            return pluginTypes;
+        }
+
+        private static void FindAndLoadPluginAssemblies()
+        {
+            var files = Directory.GetFiles(".");
+            foreach (var file in files)
+            {
+                var f = file.ToLowerInvariant();
+                if (f.Contains("plugin") && f.EndsWith("dll"))
+                {
+                    var name = Path.GetFileNameWithoutExtension(file);
+                    try
+                    {
+                        Assembly.Load(name);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new Exception($"Failed to load plugin from file '{name}'.", ex);
+                    }
+                }
+            }
+        }
+    }
+}

Framework/Core/PluginManager.cs

@@ -0,0 +1,78 @@
+namespace Core
+{
+    internal class PluginManager
+    {
+        private readonly List<PluginToolsPair> pairs = new List<PluginToolsPair>();
+
+        internal void InstantiatePlugins(Type[] pluginTypes, IToolsFactory provider)
+        {
+            pairs.Clear();
+            foreach (var pluginType in pluginTypes)
+            {
+                var tools = provider.CreateTools();
+                var plugin = InstantiatePlugins(pluginType, tools);
+
+                ApplyLogPrefix(plugin, tools);
+            }
+        }
+
+        internal void AnnouncePlugins()
+        {
+            foreach (var pair in pairs) pair.Plugin.Announce();
+        }
+
+        internal PluginMetadata GatherPluginMetadata()
+        {
+            var metadata = new PluginMetadata();
+            foreach (var pair in pairs)
+            {
+                if (pair.Plugin is IHasMetadata m)
+                {
+                    m.AddMetadata(metadata);
+                }
+            }
+            return metadata;
+        }
+
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        {
+            foreach (var pair in pairs)
+            {
+                pair.Plugin.Decommission();
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            }
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin;
+        }
+
+        private IProjectPlugin InstantiatePlugins(Type pluginType, PluginTools tools)
+        {
+            var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: tools)!;
+            pairs.Add(new PluginToolsPair(plugin, tools));
+            return plugin;
+        }
+
+        private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools)
+        {
+            if (plugin is IHasLogPrefix hasLogPrefix)
+            {
+                tools.ApplyLogPrefix(hasLogPrefix.LogPrefix);
+            }
+        }
+
+        private class PluginToolsPair
+        {
+            public PluginToolsPair(IProjectPlugin plugin, IPluginTools tools)
+            {
+                Plugin = plugin;
+                Tools = tools;
+            }
+
+            public IProjectPlugin Plugin { get; }
+            public IPluginTools Tools { get; }
+        }
+    }
+}

Framework/Core/PluginMetadata.cs

@@ -0,0 +1,27 @@
+namespace Core
+{
+    internal interface IPluginMetadata
+    {
+        Dictionary<string, string> Get();
+    }
+
+    public interface IAddMetadata
+    {
+        void Add(string key, string value);
+    }
+
+    internal class PluginMetadata : IPluginMetadata, IAddMetadata
+    {
+        private readonly Dictionary<string, string> metadata = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            metadata.Add(key, value);
+        }
+
+        public Dictionary<string, string> Get()
+        {
+            return new Dictionary<string, string>(metadata);
+        }
+    }
+}

Framework/Core/PluginTools.cs

@@ -0,0 +1,85 @@
+using FileUtils;
+using KubernetesWorkflow;
+using Logging;
+using Utils;
+
+namespace Core
+{
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    {
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+    }
+
+    public interface IWorkflowTool
+    {
+        IStartupWorkflow CreateWorkflow(string? namespaceOverride = null);
+    }
+
+    public interface ILogTool
+    {
+        ILog GetLog();
+    }
+
+    public interface IHttpFactoryTool
+    {
+        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
+        IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
+    }
+
+    public interface IFileTool
+    {
+        IFileManager GetFileManager();
+    }
+
+    internal class PluginTools : IPluginTools
+    {
+        private readonly ITimeSet timeSet;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly IFileManager fileManager;
+        private ILog log;
+
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = log;
+            this.workflowCreator = workflowCreator;
+            this.timeSet = timeSet;
+            fileManager = new FileManager(log, fileManagerRootFolder);
+        }
+
+        public void ApplyLogPrefix(string prefix)
+        {
+            log = new LogPrefixer(log, prefix);
+        }
+
+        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        {
+            return new Http(log, timeSet, address, baseUrl, onClientCreated, logAlias);
+        }
+
+        public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
+        {
+            return new Http(log, timeSet, address, baseUrl, logAlias);
+        }
+
+        public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
+        {
+            return workflowCreator.CreateWorkflow(namespaceOverride);
+        }
+
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        {
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteTrackedFiles) fileManager.DeleteAllFiles();
+        }
+
+        public IFileManager GetFileManager()
+        {
+            return fileManager;
+        }
+
+        public ILog GetLog()
+        {
+            return log;
+        }
+    }
+}

Framework/Core/ProjectPlugin.cs

@@ -0,0 +1,34 @@
+using Utils;
+
+namespace Core
+{
+    public interface IProjectPlugin
+    {
+        void Announce();
+        void Decommission();
+    }
+
+    public interface IHasLogPrefix
+    {
+        string LogPrefix { get; }
+    }
+
+    public interface IHasMetadata
+    {
+        void AddMetadata(IAddMetadata metadata);
+    }
+
+    public static class ProjectPlugin
+    {
+        /// <summary>
+        /// On some platforms and in some cases, not all required plugin assemblies are automatically loaded into the app domain.
+        /// In this case, the runtime needs a slight push to load it before the EntryPoint class is instantiated.
+        /// Used ProjectPlugin.Load<>() before you create an EntryPoint to ensure all plugins you want to use are loaded.
+        /// </summary>
+        public static void Load<T>() where T : IProjectPlugin
+        {
+            var type = typeof(T);
+            FrameworkAssert.That(type != null, $"Unable to load plugin.");
+        }
+    }
+}

Framework/Core/SerializeGate.cs

@@ -0,0 +1,20 @@
+using Newtonsoft.Json;
+
+namespace Core
+{
+    public static class SerializeGate
+    {
+        /// <summary>
+        /// SerializeGate was added to help ensure deployment objects are serializable
+        /// and remain viable after deserialization.
+        /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes
+        /// don't break this requirement.
+        /// </summary>
+        public static T Gate<T>(T anything)
+        {
+            var json = JsonConvert.SerializeObject(anything);
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}

Framework/Core/TimeSet.cs

@@ -0,0 +1,39 @@
+namespace Core
+{
+    public interface ITimeSet
+    {
+        TimeSpan HttpCallTimeout();
+        TimeSpan HttpCallRetryTime();
+        TimeSpan HttpCallRetryDelay();
+        TimeSpan WaitForK8sServiceDelay();
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
+
+        public TimeSpan HttpCallRetryTime()
+        {
+            return TimeSpan.FromMinutes(1);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(1);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+}

Framework/Core/ToolsFactory.cs

@@ -0,0 +1,31 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    internal interface IToolsFactory
+    {
+        PluginTools CreateTools();
+    }
+
+    internal class ToolsFactory : IToolsFactory
+    {
+        private readonly ILog log;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly string fileManagerRootFolder;
+        private readonly ITimeSet timeSet;
+
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = log;
+            workflowCreator = new WorkflowCreator(log, configuration);
+            this.fileManagerRootFolder = fileManagerRootFolder;
+            this.timeSet = timeSet;
+        }
+
+        public PluginTools CreateTools()
+        {
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+        }
+    }
+}

Framework/FileUtils/FileManager.cs

@@ -0,0 +1,157 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public interface IFileManager
+    {
+        TrackedFile CreateEmptyFile(string label = "");
+        TrackedFile GenerateFile(ByteSize size, string label = "");
+        void DeleteAllFiles();
+        void ScopedFiles(Action action);
+        T ScopedFiles<T>(Func<T> action);
+    }
+
+    public class FileManager : IFileManager
+    {
+        public const int ChunkSize = 1024 * 1024 * 100;
+        private static NumberSource folderNumberSource = new NumberSource(0);
+        private readonly Random random = new Random();
+        private readonly ILog log;
+        private readonly string rootFolder;
+        private readonly string folder;
+        private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
+
+        public FileManager(ILog log, string rootFolder)
+        {
+            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+
+            this.log = log;
+            this.rootFolder = rootFolder;
+        }
+
+        public TrackedFile CreateEmptyFile(string label = "")
+        {
+            var path = Path.Combine(folder, Guid.NewGuid().ToString() + ".bin");
+            EnsureDirectory();
+
+            var result = new TrackedFile(log, path, label);
+            File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
+            return result;
+        }
+
+        public TrackedFile GenerateFile(ByteSize size, string label)
+        {
+            var sw = Stopwatch.Begin(log);
+            var result = GenerateRandomFile(size, label);
+            sw.End($"Generated file '{result.Describe()}'.");
+            return result;
+        }
+
+        public void DeleteAllFiles()
+        {
+            DeleteDirectory();
+        }
+
+        public void ScopedFiles(Action action)
+        {
+            PushFileSet();
+            action();
+            PopFileSet();
+        }
+
+        public T ScopedFiles<T>(Func<T> action)
+        {
+            PushFileSet();
+            var result = action();
+            PopFileSet();
+            return result;
+        }
+
+        private void PushFileSet()
+        {
+            fileSetStack.Add(new List<TrackedFile>());
+        }
+
+        private void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                File.Delete(file.Filename);
+            }
+
+            // If the folder is now empty, delete it too.
+            if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
+        }
+
+        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        {
+            var result = CreateEmptyFile(label);
+            CheckSpaceAvailable(result, size);
+
+            GenerateFileBytes(result, size);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        {
+            var file = new FileInfo(testFile.Filename);
+            var drive = new DriveInfo(file.Directory!.Root.FullName);
+
+            var spaceAvailable = drive.TotalFreeSpace;
+
+            if (spaceAvailable < size.SizeInBytes)
+            {
+                var msg = $"Not enough disk space. " +
+                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
+
+                log.Log(msg);
+                throw new Exception(msg);
+            }
+        }
+
+        private void GenerateFileBytes(TrackedFile result, ByteSize size)
+        {
+            long bytesLeft = size.SizeInBytes;
+            int chunkSize = ChunkSize;
+            while (bytesLeft > 0)
+            {
+                try
+                {
+                    var length = Math.Min(bytesLeft, chunkSize);
+                    AppendRandomBytesToFile(result, length);
+                    bytesLeft -= length;
+                }
+                catch
+                {
+                    chunkSize = chunkSize / 2;
+                    if (chunkSize < 1024) throw;
+                }
+            }
+        }
+
+        private void AppendRandomBytesToFile(TrackedFile result, long length)
+        {
+            var bytes = new byte[length];
+            random.NextBytes(bytes);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            stream.Write(bytes, 0, bytes.Length);
+        }
+
+        private void EnsureDirectory()
+        {
+            Directory.CreateDirectory(folder);
+        }
+
+        private void DeleteDirectory()
+        {
+            if (Directory.Exists(folder)) Directory.Delete(folder, true);
+        }
+    }
+}

Framework/FileUtils/FileUtils.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/FileUtils/TrackedFile.cs

@@ -0,0 +1,82 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public class TrackedFile
+    {
+        private readonly ILog log;
+
+        public TrackedFile(ILog log, string filename, string label)
+        {
+            this.log = log;
+            Filename = filename;
+            Label = label;
+        }
+
+        public string Filename { get; }
+        public string Label { get; }
+
+        public void AssertIsEqual(TrackedFile? actual)
+        {
+            var sw = Stopwatch.Begin(log);
+            try
+            {
+                AssertEqual(actual);
+            }
+            finally
+            {
+                sw.End($"{nameof(TrackedFile)}.{nameof(AssertIsEqual)}");
+            }
+        }
+
+        public string Describe()
+        {
+            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
+            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
+            return $"'{Filename}'{sizePostfix}";
+        }
+
+        private void AssertEqual(TrackedFile? actual)
+        {
+            if (actual == null)  FrameworkAssert.Fail("TestFile is null.");
+            if (actual == this || actual!.Filename == Filename) FrameworkAssert.Fail("TestFile is compared to itself.");
+
+            FrameworkAssert.That(actual.GetFileSize() == GetFileSize(), "Files are not of equal length.");
+
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytesExpected = new byte[FileManager.ChunkSize];
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
+
+            while (true)
+            {
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
+
+                if (readExpected == 0 && readActual == 0)
+                {
+                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    return;
+                }
+
+                FrameworkAssert.That(readActual == readExpected, "Unable to read buffers of equal length.");
+
+                for (var i = 0; i < readActual; i++)
+                {
+                    if (bytesExpected[i] != bytesActual[i]) FrameworkAssert.Fail("File contents not equal.");
+                }
+            }
+        }
+
+        private long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+    }
+}

Framework/KubernetesWorkflow/ByteSizeExtensions.cs

@@ -0,0 +1,41 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public static class ByteSizeExtensions
+    {
+        public static string ToSuffixNotation(this ByteSize b)
+        {
+            long x = 1024;
+            var map = new Dictionary<long, string>
+            {
+                { Pow(x, 4), "Ti" },
+                { Pow(x, 3), "Gi" },
+                { Pow(x, 2), "Mi" },
+                { (x), "Ki" },
+            };
+
+            var bytes = b.SizeInBytes;
+            foreach (var pair in map)
+            {
+                if (bytes > pair.Key)
+                {
+                    double bytesD = bytes;
+                    double divD = pair.Key;
+                    double numD = Math.Ceiling(bytesD / divD);
+                    var v = Convert.ToInt64(numD);
+                    return $"{v}{pair.Value}";
+                }
+            }
+
+            return $"{bytes}";
+        }
+
+        private static long Pow(long x, int v)
+        {
+            long result = 1;
+            for (var i = 0; i < v; i++) result *= x;
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/CommandRunner.cs

@@ -0,0 +1,51 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class CommandRunner
+    {
+        private readonly K8sClient client;
+        private readonly string k8sNamespace;
+        private readonly RunningPod pod;
+        private readonly string containerName;
+        private readonly string command;
+        private readonly string[] arguments;
+        private readonly List<string> lines = new List<string>();
+
+        public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        {
+            this.client = client;
+            this.k8sNamespace = k8sNamespace;
+            this.pod = pod;
+            this.containerName = containerName;
+            this.command = command;
+            this.arguments = arguments;
+        }
+
+        public void Run()
+        {
+            var input = new[] { command }.Concat(arguments).ToArray();
+
+            Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
+                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+        }
+
+        public string GetStdOut()
+        {
+            return string.Join(Environment.NewLine, lines);
+        }
+
+        private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr)
+        {
+            using var streamReader = new StreamReader(stdOut);
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                lines.Add(line);
+                line = streamReader.ReadLine();
+            }
+
+            return Task.CompletedTask;
+        }
+    }
+}

Framework/KubernetesWorkflow/Configuration.cs

@@ -0,0 +1,25 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow
+{
+    public class Configuration
+    {
+        public Configuration(string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, string kubernetesNamespace)
+        {
+            KubeConfigFile = kubeConfigFile;
+            OperationTimeout = operationTimeout;
+            RetryDelay = retryDelay;
+            KubernetesNamespace = kubernetesNamespace;
+        }
+
+        public string? KubeConfigFile { get; }
+        public TimeSpan OperationTimeout { get; }
+        public TimeSpan RetryDelay { get; }
+        public string KubernetesNamespace { get; }
+        public bool AllowNamespaceOverride { get; set; } = true;
+        public bool AddAppPodLabel { get; set; } = true;
+
+        [JsonIgnore]
+        public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
+    }
+}

Framework/KubernetesWorkflow/ContainerAdditionals.cs

@@ -0,0 +1,53 @@
+using Newtonsoft.Json.Linq;
+
+namespace KubernetesWorkflow
+{
+    public class ContainerAdditionals
+    {
+        public ContainerAdditionals(Additional[] additionals)
+        {
+            Additionals = additionals;
+        }
+
+        public static ContainerAdditionals CreateFromUserData(IEnumerable<object> userData)
+        {
+            return new ContainerAdditionals(userData.Select(ConvertToAdditional).ToArray());
+        }
+
+        public Additional[] Additionals { get; }
+
+        public T? Get<T>()
+        {
+            var typeName = GetTypeName(typeof(T));
+            var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
+            if (userData == null) return default(T);
+            var jobject = (JObject)userData.UserData;
+            return jobject.ToObject<T>();
+        }
+
+        private static Additional ConvertToAdditional(object userData)
+        {
+            var typeName = GetTypeName(userData.GetType());
+            return new Additional(typeName, userData);
+        }
+
+        private static string GetTypeName(Type type)
+        {
+            var typeName = type.FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Object type fullname is null or empty: " + type);
+            return typeName;
+        }
+    }
+
+    public class Additional
+    {
+        public Additional(string type, object userData)
+        {
+            Type = type;
+            UserData = userData;
+        }
+
+        public string Type { get; }
+        public object UserData { get; }
+    }
+}

Framework/KubernetesWorkflow/ContainerRecipe.cs

@@ -0,0 +1,100 @@
+namespace KubernetesWorkflow
+{
+    public class ContainerRecipe
+    {
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        {
+            Number = number;
+            NameOverride = nameOverride;
+            Image = image;
+            Resources = resources;
+            ExposedPorts = exposedPorts;
+            InternalPorts = internalPorts;
+            EnvVars = envVars;
+            PodLabels = podLabels;
+            PodAnnotations = podAnnotations;
+            Volumes = volumes;
+            Additionals = additionals;
+
+            if (NameOverride != null)
+            {
+                Name = $"{K8sNameUtils.Format(NameOverride)}-{Number}";
+            }
+            else
+            {
+                Name = $"ctnr{Number}";
+            }
+        }
+
+        public string Name { get; }
+        public int Number { get; }
+        public string? NameOverride { get; }
+        public ContainerResources Resources { get; }
+        public string Image { get; }
+        public Port[] ExposedPorts { get; }
+        public Port[] InternalPorts { get; }
+        public EnvVar[] EnvVars { get; }
+        public PodLabels PodLabels { get; }
+        public PodAnnotations PodAnnotations { get; }
+        public VolumeMount[] Volumes { get; }
+        public ContainerAdditionals Additionals { get; }
+
+        public Port? GetPortByTag(string tag)
+        {
+            return ExposedPorts.Concat(InternalPorts).SingleOrDefault(p => p.Tag == tag);
+        }
+
+        public override string ToString()
+        {
+            return $"(container-recipe: {Name}, image: {Image}, " +
+                $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
+                $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
+                $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
+                $"limits: {Resources}, " +
+                $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
+        }
+    }
+
+    public class Port
+    {
+        public Port(int number, string tag)
+        {
+            Number = number;
+            Tag = tag;
+        }
+
+        public int Number { get; }
+        public string Tag { get; }
+    }
+
+    public class EnvVar
+    {
+        public EnvVar(string name, string value)
+        {
+            Name = name;
+            Value = value;
+        }
+
+        public string Name { get; }
+        public string Value { get; }
+
+        public override string ToString()
+        {
+            return $"'{Name}' = '{Value}'";
+        }
+    }
+
+    public class VolumeMount
+    {
+        public VolumeMount(string volumeName, string mountPath, string resourceQuantity)
+        {
+            VolumeName = volumeName;
+            MountPath = mountPath;
+            ResourceQuantity = resourceQuantity;
+        }
+
+        public string VolumeName { get; }
+        public string MountPath { get; }
+        public string ResourceQuantity { get; }
+    }
+}

Framework/KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -0,0 +1,144 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public abstract class ContainerRecipeFactory
+    {
+        private readonly List<Port> exposedPorts = new List<Port>();
+        private readonly List<Port> internalPorts = new List<Port>();
+        private readonly List<EnvVar> envVars = new List<EnvVar>();
+        private readonly PodLabels podLabels = new PodLabels();
+        private readonly PodAnnotations podAnnotations = new PodAnnotations();
+        private readonly List<VolumeMount> volumeMounts = new List<VolumeMount>();
+        private readonly List<object> additionals = new List<object>();
+        private RecipeComponentFactory factory = null!;
+        private ContainerResources resources = new ContainerResources();
+
+        public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
+        {
+            this.factory = factory;
+            ContainerNumber = containerNumber;
+            Index = index;
+
+            Initialize(config);
+
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+                exposedPorts.ToArray(),
+                internalPorts.ToArray(),
+                envVars.ToArray(),
+                podLabels.Clone(),
+                podAnnotations.Clone(),
+                volumeMounts.ToArray(),
+                ContainerAdditionals.CreateFromUserData(additionals));
+
+            exposedPorts.Clear();
+            internalPorts.Clear();
+            envVars.Clear();
+            podLabels.Clear();
+            podAnnotations.Clear();
+            volumeMounts.Clear();
+            additionals.Clear();
+            this.factory = null!;
+            resources = new ContainerResources();
+
+            return recipe;
+        }
+
+        public abstract string AppName { get; }
+        public abstract string Image { get; }
+        protected int ContainerNumber { get; private set; } = 0;
+        protected int Index { get; private set; } = 0;
+        protected abstract void Initialize(StartupConfig config);
+
+        protected Port AddExposedPort(string tag = "")
+        {
+            return AddExposedPort(factory.CreatePort(tag));
+        }
+
+        protected Port AddExposedPort(int number, string tag = "")
+        {
+            return AddExposedPort(factory.CreatePort(number, tag));
+        }
+
+        protected Port AddInternalPort(string tag = "")
+        {
+            var p = factory.CreatePort(tag);
+            internalPorts.Add(p);
+            return p;
+        }
+
+        protected void AddExposedPortAndVar(string name, string tag = "")
+        {
+            AddEnvVar(name, AddExposedPort(tag));
+        }
+
+        protected void AddInternalPortAndVar(string name, string tag = "")
+        {
+            AddEnvVar(name, AddInternalPort(tag));
+        }
+
+        protected void AddEnvVar(string name, string value)
+        {
+            envVars.Add(factory.CreateEnvVar(name, value));
+        }
+
+        protected void AddEnvVar(string name, Port value)
+        {
+            envVars.Add(factory.CreateEnvVar(name, value.Number));
+        }
+
+        protected void AddPodLabel(string name, string value)
+        {
+            podLabels.Add(name, value);
+        }
+
+        protected void AddPodAnnotation(string name, string value)
+        {
+            podAnnotations.Add(name, value);
+        }
+
+        protected void AddVolume(string mountPath, ByteSize volumeSize)
+        {
+            volumeMounts.Add(new VolumeMount(
+                $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
+                mountPath,
+                volumeSize.ToSuffixNotation()));
+        }
+
+        protected void Additional(object userData)
+        {
+            additionals.Add(userData);
+        }
+
+        protected void SetResourcesRequest(int milliCPUs, ByteSize memory)
+        {
+            SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
+        }
+
+        protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        {
+            SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        }
+
+        protected void SetResourcesRequest(ContainerResourceSet requests)
+        {
+            resources.Requests = requests;
+        }
+
+        protected void SetResourceLimits(ContainerResourceSet limits)
+        {
+            resources.Limits = limits;
+        }
+
+        private Port AddExposedPort(Port port)
+        {
+            if (exposedPorts.Any())
+            {
+                throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
+                    $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
+            }
+            exposedPorts.Add(port);
+            return port;
+        }
+    }
+}

Framework/KubernetesWorkflow/ContainerResources.cs

@@ -0,0 +1,52 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class ContainerResources
+    {
+        public ContainerResourceSet Requests { get; set; } = new ContainerResourceSet();
+        public ContainerResourceSet Limits { get; set; } = new ContainerResourceSet();
+
+        public override string ToString()
+        {
+            return $"requests:{Requests}, limits:{Limits}";
+        }
+    }
+
+    public class ContainerResourceSet
+    {
+        public ContainerResourceSet(int milliCPUs, ByteSize memory)
+        {
+            MilliCPUs = milliCPUs;
+            Memory = memory;
+        }
+
+        public ContainerResourceSet(int milliCPUs)
+            : this(milliCPUs, new ByteSize(0))
+        {
+        }
+
+        public ContainerResourceSet(ByteSize memory)
+            : this(0, memory)
+        {
+        }
+
+        public ContainerResourceSet()
+            : this(0)
+        {
+        }
+
+        public int MilliCPUs { get; }
+        public ByteSize Memory { get; }
+
+        public override string ToString()
+        {
+            var result = new List<string>();
+            if (MilliCPUs == 0) result.Add("cpu: unlimited");
+            else result.Add($"cpu: {MilliCPUs} milliCPUs");
+            if (Memory.SizeInBytes == 0) result.Add("memory: unlimited");
+            else result.Add($"memory: {Memory}");
+            return string.Join(", ", result);
+        }
+    }
+}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -0,0 +1,95 @@
+using k8s;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class CrashWatcher
+    {
+        private readonly ILog log;
+        private readonly KubernetesClientConfiguration config;
+        private readonly string k8sNamespace;
+        private readonly RunningContainer container;
+        private ILogHandler? logHandler;
+        private CancellationTokenSource cts;
+        private Task? worker;
+        private Exception? workerException;
+
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        {
+            this.log = log;
+            this.config = config;
+            this.k8sNamespace = k8sNamespace;
+            this.container = container;
+            cts = new CancellationTokenSource();
+        }
+
+        public void Start(ILogHandler logHandler)
+        {
+            if (worker != null) throw new InvalidOperationException();
+
+            this.logHandler = logHandler;
+            cts = new CancellationTokenSource();
+            worker = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            if (worker == null) throw new InvalidOperationException();
+
+            cts.Cancel();
+            worker.Wait();
+            worker = null;
+
+            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
+        }
+
+        public bool HasContainerCrashed()
+        {
+            using var client = new Kubernetes(config);
+            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
+        }
+
+        private void Worker()
+        {
+            try
+            {
+                MonitorContainer(cts.Token);
+            }
+            catch (Exception ex)
+            {
+                workerException = ex;
+            }
+        }
+
+        private void MonitorContainer(CancellationToken token)
+        {
+            using var client = new Kubernetes(config);
+            while (!token.IsCancellationRequested)
+            {
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+
+                var pod = container.Pod;
+                var recipe = container.Recipe;
+                var podName = pod.PodInfo.Name;
+                if (HasContainerBeenRestarted(client, podName))
+                {
+                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    return;
+                }
+            }
+        }
+
+        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
+        {
+            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
+            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+        }
+
+        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        {
+            log.Log("Pod crash detected for " + container.Name);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            logHandler!.Log(stream);
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sClient.cs

@@ -0,0 +1,36 @@
+using k8s;
+
+namespace KubernetesWorkflow
+{
+    public class K8sClient
+    {
+        private readonly Kubernetes client;
+        private static readonly object clientLock = new object();
+
+        public K8sClient(KubernetesClientConfiguration config)
+        {
+            client = new Kubernetes(config);
+        }
+
+        public void Run(Action<Kubernetes> action)
+        {
+            lock (clientLock)
+            {
+                action(client);
+            }
+        }
+
+        public T Run<T>(Func<Kubernetes, T> action)
+        {
+            lock (clientLock)
+            {
+                return action(client);
+            }
+        }
+
+        public void Dispose()
+        {
+            client.Dispose();
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -0,0 +1,69 @@
+using k8s;
+
+namespace KubernetesWorkflow
+{
+    public class K8sCluster
+    {
+        public K8sCluster(Configuration configuration)
+        {
+            Configuration = configuration;
+        }
+
+        public Configuration Configuration { get; }
+        public string HostAddress { get; private set; } = string.Empty;
+
+        public KubernetesClientConfiguration GetK8sClientConfig()
+        {
+            var config = GetConfig();
+            UpdateHostAddress(config);
+            return config;
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return Configuration.OperationTimeout;
+        }
+
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return Configuration.RetryDelay;
+        }
+
+        private KubernetesClientConfiguration GetConfig()
+        {
+            if (Configuration.KubeConfigFile != null)
+            {
+                return KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile);
+            }
+            else
+            {
+                return KubernetesClientConfiguration.BuildDefaultConfig();
+            }
+        }
+
+        private void UpdateHostAddress(KubernetesClientConfiguration config)
+        {
+            var host = config.Host.Replace("https://", "");
+            if (host.Contains(":"))
+            {
+                HostAddress = "http://" + host.Substring(0, host.IndexOf(':'));
+            }
+            else
+            {
+                HostAddress = config.Host;
+            }
+        }
+    }
+
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
+    }
+}

Framework/KubernetesWorkflow/K8sController.cs

@@ -0,0 +1,727 @@
+using k8s;
+using k8s.Models;
+using Logging;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class K8sController
+    {
+        private readonly ILog log;
+        private readonly K8sCluster cluster;
+        private readonly KnownK8sPods knownPods;
+        private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly K8sClient client;
+
+        public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        {
+            this.log = log;
+            this.cluster = cluster;
+            this.knownPods = knownPods;
+            this.workflowNumberSource = workflowNumberSource;
+            client = new K8sClient(cluster.GetK8sClientConfig());
+
+            K8sNamespace = k8sNamespace;
+        }
+
+        public void Dispose()
+        {
+            client.Dispose();
+        }
+
+        public RunningPod BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
+        {
+            log.Debug();
+            EnsureNamespace();
+
+            var deploymentName = CreateDeployment(containerRecipes, location);
+            var (serviceName, servicePortsMap) = CreateService(containerRecipes);
+            var podInfo = FetchNewPod();
+
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
+        }
+
+        public void Stop(RunningPod pod)
+        {
+            log.Debug();
+            if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
+            DeleteDeployment(pod.DeploymentName);
+            WaitUntilDeploymentOffline(pod.DeploymentName);
+            WaitUntilPodOffline(pod.PodInfo.Name);
+        }
+
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
+        {
+            log.Debug();
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sNamespace, recipe.Name, tailLines: tailLines));
+            logHandler.Log(stream);
+        }
+
+        public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
+        {
+            var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
+            log.Debug(cmdAndArgs);
+
+            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            runner.Run();
+            var result = runner.GetStdOut();
+
+            log.Debug($"{cmdAndArgs} = '{result}'");
+            return result;
+        }
+
+        public void DeleteAllNamespacesStartingWith(string prefix)
+        {
+            log.Debug();
+
+            var all = client.Run(c => c.ListNamespace().Items);
+            var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
+
+            foreach (var ns in namespaces)
+            {
+                DeleteNamespace(ns);
+            }
+            foreach (var ns in namespaces)
+            {
+                WaitUntilNamespaceDeleted(ns);
+            }
+        }
+
+        public void DeleteNamespace()
+        {
+            log.Debug();
+            if (IsNamespaceOnline(K8sNamespace))
+            {
+                client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+            }
+            WaitUntilNamespaceDeleted();
+        }
+
+        public void DeleteNamespace(string ns)
+        {
+            log.Debug();
+            if (IsNamespaceOnline(ns))
+            {
+                client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+            }
+        }
+
+        #region Discover K8s Nodes
+
+        public K8sNodeLabel[] GetAvailableK8sNodes()
+        {
+            var nodes = client.Run(c => c.ListNode());
+
+            var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
+            return optionals.Where(n => n != null).Select(n => n!).ToArray();
+        }
+
+        private K8sNodeLabel? CreateNodeLabel(V1Node i)
+        {
+            var keys = i.Metadata.Labels.Keys;
+            var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
+            if (hostnameKey != null)
+            {
+                var hostnameValue = i.Metadata.Labels[hostnameKey];
+                return new K8sNodeLabel(hostnameKey, hostnameValue);
+            }
+            return null;
+        }
+
+        #endregion
+
+        #region Namespace management
+
+        private string K8sNamespace { get; }
+
+        private void EnsureNamespace()
+        {
+            if (IsNamespaceOnline(K8sNamespace)) return;
+
+            var namespaceSpec = new V1Namespace
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = K8sNamespace,
+                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
+                }
+            };
+            client.Run(c => c.CreateNamespace(namespaceSpec));
+            WaitUntilNamespaceCreated();
+
+            CreatePolicy();
+        }
+
+        private bool IsNamespaceOnline(string name)
+        {
+            return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
+        }
+
+        private void CreatePolicy()
+        {
+            client.Run(c =>
+            {
+                var body = new V1NetworkPolicy
+                {
+                    Metadata = new V1ObjectMeta
+                    {
+                        Name = "isolate-policy",
+                        NamespaceProperty = K8sNamespace
+                    },
+                    Spec = new V1NetworkPolicySpec
+                    {
+                        PodSelector = new V1LabelSelector {},
+                        PolicyTypes = new[]
+                        {
+                            "Ingress",
+                            "Egress"
+                        },
+                        Ingress = new List<V1NetworkPolicyIngressRule>
+                        {
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetRunnerNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetPrometheusNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        Egress = new List<V1NetworkPolicyEgressRule>
+                        {
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = new Dictionary<string, string> { { "kubernetes.io/metadata.name", "kube-system" } }
+                                        }
+                                    },
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = new Dictionary<string, string> { { "k8s-app", "kube-dns" } }
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "53"
+                                        },
+                                        Protocol = "UDP"
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        IpBlock = new V1IPBlock
+                                        {
+                                            Cidr = "0.0.0.0/0"
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "80"
+                                        },
+                                        Protocol = "TCP"
+                                    },
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "443"
+                                        },
+                                        Protocol = "TCP"
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+                };
+
+                c.CreateNamespacedNetworkPolicy(body, K8sNamespace);
+            });
+        }
+
+        #endregion
+
+        #region Deployment management
+
+        private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location)
+        {
+            var deploymentSpec = new V1Deployment
+            {
+                ApiVersion = "apps/v1",
+                Metadata = CreateDeploymentMetadata(containerRecipes),
+                Spec = new V1DeploymentSpec
+                {
+                    Replicas = 1,
+                    Selector = new V1LabelSelector
+                    {
+                        MatchLabels = GetSelector(containerRecipes)
+                    },
+                    Template = new V1PodTemplateSpec
+                    {
+                        Metadata = new V1ObjectMeta
+                        {
+                            Labels = GetSelector(containerRecipes),
+                            Annotations = GetAnnotations(containerRecipes)
+                        },
+                        Spec = new V1PodSpec
+                        {
+                            NodeSelector = CreateNodeSelector(location),
+                            Containers = CreateDeploymentContainers(containerRecipes),
+                            Volumes = CreateVolumes(containerRecipes)
+                        }
+                    }
+                }
+            };
+
+            client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
+            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
+
+            return deploymentSpec.Metadata.Name;
+        }
+
+        private void DeleteDeployment(string deploymentName)
+        {
+            client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sNamespace));
+            WaitUntilDeploymentOffline(deploymentName);
+        }
+
+        private IDictionary<string, string> CreateNodeSelector(ILocation location)
+        {
+            var nodeLabel = GetNodeLabelForLocation(location);
+            if (nodeLabel == null) return new Dictionary<string, string>();
+
+            return new Dictionary<string, string>
+            {
+                { nodeLabel.Key, nodeLabel.Value }
+            };
+        }
+
+        private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
+        {
+            var l = (Location)location;
+            return l.NodeLabel;
+        }
+
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodLabels.GetLabels();
+        }
+
+        private IDictionary<string, string> GetRunnerNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
+        }
+
+        private IDictionary<string, string> GetPrometheusNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "monitoring" } };
+        }
+
+        private IDictionary<string, string> GetAnnotations(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodAnnotations.GetAnnotations();
+        }
+
+        private V1ObjectMeta CreateDeploymentMetadata(ContainerRecipe[] containerRecipes)
+        {
+            return new V1ObjectMeta
+            {
+                Name = string.Join('-',containerRecipes.Select(r => r.Name)),
+                NamespaceProperty = K8sNamespace,
+                Labels = GetSelector(containerRecipes),
+                Annotations = GetAnnotations(containerRecipes)
+            };
+        }
+
+        private List<V1Container> CreateDeploymentContainers(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Select(CreateDeploymentContainer).ToList();
+        }
+
+        private V1Container CreateDeploymentContainer(ContainerRecipe recipe)
+        {
+            return new V1Container
+            {
+                Name = recipe.Name,
+                Image = recipe.Image,
+                ImagePullPolicy = "Always",
+                Ports = CreateContainerPorts(recipe),
+                Env = CreateEnv(recipe),
+                VolumeMounts = CreateContainerVolumeMounts(recipe),
+                Resources = CreateResourceLimits(recipe)
+            };
+        }
+
+        private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe)
+        {
+            return new V1ResourceRequirements
+            {
+                Requests = CreateResourceQuantities(recipe.Resources.Requests),
+                Limits = CreateResourceQuantities(recipe.Resources.Limits)
+            };
+        }
+
+        private Dictionary<string, ResourceQuantity> CreateResourceQuantities(ContainerResourceSet set)
+        {
+            var result = new Dictionary<string, ResourceQuantity>();
+            if (set.MilliCPUs != 0)
+            {
+                result.Add("cpu", new ResourceQuantity($"{set.MilliCPUs}m"));
+            }
+            if (set.Memory.SizeInBytes != 0)
+            {
+                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+            }
+            return result;
+        }
+
+        private List<V1VolumeMount> CreateContainerVolumeMounts(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateContainerVolumeMount).ToList();
+        }
+
+        private V1VolumeMount CreateContainerVolumeMount(VolumeMount v)
+        {
+            return new V1VolumeMount
+            {
+                Name = v.VolumeName,
+                MountPath = v.MountPath
+            };
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList();
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateVolume).ToList();
+        }
+
+        private V1Volume CreateVolume(VolumeMount v)
+        {
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteOnce"
+                    },
+                    Resources = new V1ResourceRequirements
+                    {
+                        Requests = new Dictionary<string, ResourceQuantity>
+                        {
+                            {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                        }
+                    }
+                }
+            }, K8sNamespace));
+
+            return new V1Volume
+            {
+                Name = v.VolumeName,
+                PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource
+                {
+                    ClaimName = v.VolumeName
+                }
+            };
+        }
+
+        private List<V1EnvVar> CreateEnv(ContainerRecipe recipe)
+        {
+            return recipe.EnvVars.Select(CreateEnvVar).ToList();
+        }
+
+        private V1EnvVar CreateEnvVar(EnvVar envVar)
+        {
+            return new V1EnvVar
+            {
+                Name = envVar.Name,
+                Value = envVar.Value,
+            };
+        }
+
+        private List<V1ContainerPort> CreateContainerPorts(ContainerRecipe recipe)
+        {
+            var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p));
+            var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p));
+            return exposedPorts.Concat(internalPorts).ToList();
+        }
+
+        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port)
+        {
+            return new V1ContainerPort
+            {
+                Name = GetNameForPort(recipe, port),
+                ContainerPort = port.Number
+            };
+        }
+
+        private string GetNameForPort(ContainerRecipe recipe, Port port)
+        {
+            return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}";
+        }
+
+        #endregion
+
+        #region Service management
+
+        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
+        {
+            var result = new List<ContainerRecipePortMapEntry>();
+
+            var ports = CreateServicePorts(containerRecipes);
+
+            if (!ports.Any())
+            {
+                // None of these container-recipes wish to expose anything via a service port.
+                // So, we don't have to create a service.
+                return (string.Empty, result);
+            }
+
+            var serviceSpec = new V1Service
+            {
+                ApiVersion = "v1",
+                Metadata = CreateServiceMetadata(),
+                Spec = new V1ServiceSpec
+                {
+                    Type = "NodePort",
+                    Selector = GetSelector(containerRecipes),
+                    Ports = ports
+                }
+            };
+
+            client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
+
+            ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
+
+            return (serviceSpec.Metadata.Name, result);
+        }
+
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
+        {
+            // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
+            var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
+            foreach (var r in containerRecipes)
+            {
+                if (r.ExposedPorts.Any())
+                {
+                    var firstExposedPort = r.ExposedPorts.First();
+                    var portName = GetNameForPort(r, firstExposedPort);
+
+                    var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
+                    if (matchingServicePorts.Any())
+                    {
+                        // These service ports belongs to this recipe.
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
+
+                        result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
+                    }
+                }
+            }
+        }
+
+        private Port? MapNodePortIfAble(V1ServicePort p, string tag)
+        {
+            if (p.NodePort == null) return null;
+            return new Port(p.NodePort.Value, tag);
+        }
+
+        private void DeleteService(string serviceName)
+        {
+            client.Run(c => c.DeleteNamespacedService(serviceName, K8sNamespace));
+        }
+
+        private V1ObjectMeta CreateServiceMetadata()
+        {
+            return new V1ObjectMeta
+            {
+                Name = "service-" + workflowNumberSource.WorkflowNumber,
+                NamespaceProperty = K8sNamespace
+            };
+        }
+
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
+        {
+            var result = new List<V1ServicePort>();
+            foreach (var recipe in recipes)
+            {
+                result.AddRange(CreateServicePorts(recipe));
+            }
+            return result;
+        }
+
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
+        {
+            var result = new List<V1ServicePort>();
+            foreach (var port in recipe.ExposedPorts)
+            {
+                result.Add(new V1ServicePort
+                {
+                    Name = GetNameForPort(recipe, port),
+                    Protocol = "TCP",
+                    Port = port.Number,
+                    TargetPort = GetNameForPort(recipe, port),
+                });                
+            }
+
+            return result;
+        }
+
+        #endregion
+
+        #region Waiting
+
+        private void WaitUntilNamespaceCreated() 
+        {
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
+        }
+
+        private void WaitUntilNamespaceDeleted()
+        {
+            WaitUntil(() => !IsNamespaceOnline(K8sNamespace));
+        }
+
+        private void WaitUntilNamespaceDeleted(string name)
+        {
+            WaitUntil(() => !IsNamespaceOnline(name));
+        }
+
+        private void WaitUntilDeploymentOnline(string deploymentName)
+        {
+            WaitUntil(() =>
+            {
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
+                return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
+            });
+        }
+
+        private void WaitUntilDeploymentOffline(string deploymentName)
+        {
+            WaitUntil(() =>
+            {
+                var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
+                var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
+                return deployment == null || deployment.Status.AvailableReplicas == 0;
+            });
+        }
+
+        private void WaitUntilPodOffline(string podName)
+        {
+            WaitUntil(() =>
+            {
+                var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
+                var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
+                return pod == null;
+            });
+        }
+
+        private void WaitUntil(Func<bool> predicate)
+        {
+            var sw = Stopwatch.Begin(log, true);
+            try
+            {
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
+            }
+            finally
+            {
+                sw.End("", 1);
+            }
+        }
+
+        #endregion
+
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
+        }
+
+        private PodInfo FetchNewPod()
+        {
+            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
+
+            var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
+            if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
+
+            var newPod = newPods.Single();
+            var name = newPod.Name();
+            var ip = newPod.Status.PodIP;
+            var k8sNodeName = newPod.Spec.NodeName;
+
+            if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
+            if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
+
+            knownPods.Add(name);
+            return new PodInfo(name, ip, k8sNodeName);
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -0,0 +1,24 @@
+namespace KubernetesWorkflow
+{
+    public interface IK8sHooks
+    {
+        void OnContainersStarted(RunningContainers runningContainers);
+        void OnContainersStopped(RunningContainers runningContainers);
+        void OnContainerRecipeCreated(ContainerRecipe recipe);
+    }
+
+    public class DoNothingK8sHooks : IK8sHooks
+    {
+        public void OnContainersStarted(RunningContainers runningContainers)
+        {
+        }
+
+        public void OnContainersStopped(RunningContainers runningContainers)
+        {
+        }
+
+        public void OnContainerRecipeCreated(ContainerRecipe recipe)
+        {
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -0,0 +1,22 @@
+namespace KubernetesWorkflow
+{
+    public static class K8sNameUtils
+    {
+        public static string Format(string s)
+        {
+            var result = s.ToLowerInvariant()
+                .Replace(" ", "-")
+                .Replace(":", "-")
+                .Replace("/", "-")
+                .Replace("\\", "-")
+                .Replace("[", "-")
+                .Replace("]", "-")
+                .Replace(",", "-");
+
+            result = result.Trim('-');
+            if (result.Length > 62) result = result.Substring(0, 62);
+
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/KnownK8sPods.cs

@@ -1,4 +1,4 @@
-namespace CodexDistTestCore
+namespace KubernetesWorkflow
 {
     public class KnownK8sPods
     {

Framework/KubernetesWorkflow/KnownLocations.cs

@@ -0,0 +1,47 @@
+namespace KubernetesWorkflow
+{
+    public interface IKnownLocations
+    {
+        /// <summary>
+        /// Returns a known location given an index.
+        /// Each index guarantees a different location.
+        /// </summary>
+        ILocation Get(int index);
+        int NumberOfLocations { get; }
+
+        /// <summary>
+        /// Returns the location object for a specific kubernetes node. Throws if it doesn't exist.
+        /// </summary>
+        ILocation Get(string kubeNodeName, bool allowPartialMatches = false);
+    }
+
+    public class KnownLocations : IKnownLocations
+    {
+        private readonly Location[] locations;
+
+        public KnownLocations(Location[] locations)
+        {
+            this.locations = locations;
+            if (locations.Any(l => l.NodeLabel == null)) throw new Exception("Must not contain unspecified location");
+        }
+
+        public static ILocation UnspecifiedLocation { get; } = new Location();
+
+        public int NumberOfLocations => locations.Length;
+
+        public ILocation Get(int index)
+        {
+            return locations[index];
+        }
+
+        public ILocation Get(string kubeNodeName, bool allowPartialMatches = false)
+        {
+            if (allowPartialMatches)
+            {
+                return locations.Single(l => l.NodeLabel != null && l.NodeLabel.Value.Contains(kubeNodeName));
+            }
+
+            return locations.Single(l => l.NodeLabel != null && l.NodeLabel.Value == kubeNodeName);
+        }
+    }
+}

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+	  <RootNamespace>KubernetesWorkflow</RootNamespace>
+	  <ImplicitUsings>enable</ImplicitUsings>
+	  <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+	  <PackageReference Include="KubernetesClient" Version="10.1.4" />
+	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/KubernetesWorkflow/Location.cs

@@ -0,0 +1,22 @@
+namespace KubernetesWorkflow
+{
+    public interface ILocation
+    {
+    }
+
+    public class Location : ILocation
+    {
+        internal Location(K8sNodeLabel? nodeLabel = null)
+        {
+            NodeLabel = nodeLabel;
+        }
+
+        internal K8sNodeLabel? NodeLabel { get; }
+
+        public override string ToString()
+        {
+            if (NodeLabel == null) return "Location:Unspecified";
+            return $"Location:KubeNode-'{NodeLabel.Key}:{NodeLabel.Value}'";
+        }
+    }
+}

Framework/KubernetesWorkflow/LocationProvider.cs

@@ -0,0 +1,49 @@
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class LocationProvider
+    {
+        private readonly TimeSpan locationsExpirationTime = TimeSpan.FromMinutes(10);
+        private readonly ILog log;
+        private readonly Action<Action<K8sController>> onController;
+        private Location[] knownLocations = Array.Empty<Location>();
+        private DateTime lastUpdate = DateTime.UtcNow;
+
+        public LocationProvider(ILog log, Action<Action<K8sController>> onController)
+        {
+            this.log = log;
+            this.onController = onController;
+        }
+
+        public IKnownLocations GetAvailableLocations()
+        {
+            if (ShouldUpdateKnownLocations())
+            {
+                onController(UpdateKnownLocations);
+            }
+
+            return new KnownLocations(knownLocations);
+        }
+
+        private void UpdateKnownLocations(K8sController controller)
+        {
+            knownLocations = controller.GetAvailableK8sNodes().Select(CreateLocation).ToArray();
+            lastUpdate = DateTime.UtcNow;
+
+            log.Log($"Detected {knownLocations.Length} available locations: '{string.Join(",", knownLocations.Select(l => l.ToString()))}'");
+        }
+
+        private Location CreateLocation(K8sNodeLabel k8sNode)
+        {
+            return new Location(k8sNode);
+        }
+
+        private bool ShouldUpdateKnownLocations()
+        {
+            if (!knownLocations.Any()) return true;
+            if (DateTime.UtcNow - lastUpdate > locationsExpirationTime) return true;
+            return false;
+        }
+    }
+}

Framework/KubernetesWorkflow/PodAnnotations.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow
+{
+    public class PodAnnotations
+    {
+        private readonly Dictionary<string, string> annotations = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            annotations.Add(key, value);
+        }
+
+        public PodAnnotations Clone()
+        {
+            var result = new PodAnnotations();
+            foreach (var entry in annotations) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            annotations.Clear();
+        }
+
+        internal Dictionary<string, string> GetAnnotations()
+        {
+            return annotations;
+        }
+    }
+}

Framework/KubernetesWorkflow/PodLabels.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow
+{
+    public class PodLabels
+    {
+        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            labels.Add(key, K8sNameUtils.Format(value));
+        }
+
+        public PodLabels Clone()
+        {
+            var result = new PodLabels();
+            foreach (var entry in labels) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            labels.Clear();
+        }
+
+        internal Dictionary<string, string> GetLabels()
+        {
+            return labels;
+        }
+    }
+}

Framework/KubernetesWorkflow/RecipeComponentFactory.cs

@@ -0,0 +1,30 @@
+using System.Globalization;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class RecipeComponentFactory
+    {
+        private NumberSource portNumberSource = new NumberSource(8080);
+
+        public Port CreatePort(int number, string tag)
+        {
+            return new Port(number, tag);
+        }
+
+        public Port CreatePort(string tag)
+        {
+            return new Port(portNumberSource.GetNextNumber(), tag);
+        }
+
+        public EnvVar CreateEnvVar(string name, int value)
+        {
+            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
+        }
+
+        public EnvVar CreateEnvVar(string name, string value)
+        {
+            return new EnvVar(name, value);
+        }
+    }
+}

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -0,0 +1,55 @@
+using System.Net.NetworkInformation;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    internal enum RunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
+    }
+
+    internal static class RunnerLocationUtils
+    {
+        private static RunnerLocation? knownLocation = null;
+
+        internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            if (knownLocation != null) return knownLocation.Value;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                knownLocation = RunnerLocation.InternalToCluster;
+            }
+            else if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                knownLocation = RunnerLocation.ExternalToCluster;
+            }
+
+            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
+            return knownLocation.Value;
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
+}

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -0,0 +1,70 @@
+using Newtonsoft.Json;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class RunningContainers
+    {
+        public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers)
+        {
+            StartupConfig = startupConfig;
+            RunningPod = runningPod;
+            Containers = containers;
+        }
+
+        public StartupConfig StartupConfig { get; }
+        public RunningPod RunningPod { get; }
+        public RunningContainer[] Containers { get; }
+
+        public string Describe()
+        {
+            return string.Join(",", Containers.Select(c => c.Name));
+        }
+    }
+
+    public class RunningContainer
+    {
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
+        {
+            Pod = pod;
+            Recipe = recipe;
+            ServicePorts = servicePorts;
+            Name = name;
+            ClusterExternalAddress = clusterExternalAddress;
+            ClusterInternalAddress = clusterInternalAddress;
+        }
+
+        public string Name { get; }
+        public RunningPod Pod { get; }
+        public ContainerRecipe Recipe { get; }
+        public Port[] ServicePorts { get; }
+        public Address ClusterExternalAddress { get; }
+        public Address ClusterInternalAddress { get; }
+
+        [JsonIgnore]
+        public Address Address
+        {
+            get
+            {
+                if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
+                {
+                    return ClusterInternalAddress;
+                }
+                return ClusterExternalAddress;
+            }
+        }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
+}

Framework/KubernetesWorkflow/RunningPod.cs

@@ -0,0 +1,56 @@
+namespace KubernetesWorkflow
+{
+    public class RunningPod
+    {
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
+        {
+            Cluster = cluster;
+            PodInfo = podInfo;
+            DeploymentName = deploymentName;
+            ServiceName = serviceName;
+            PortMapEntries = portMapEntries;
+        }
+
+        public K8sCluster Cluster { get; }
+        public PodInfo PodInfo { get; }
+        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
+        public string DeploymentName { get; }
+        public string ServiceName { get; }
+
+        public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
+        {
+            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
+            {
+                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
+            }
+
+            return Array.Empty<Port>();
+        }
+    }
+
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
+        {
+            ContainerNumber = containerNumber;
+            Ports = ports;
+        }
+
+        public int ContainerNumber { get; }
+        public Port[] Ports { get; }
+    }
+
+    public class PodInfo
+    {
+        public PodInfo(string name, string ip, string k8sNodeName)
+        {
+            Name = name;
+            Ip = ip;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
+}

Framework/KubernetesWorkflow/StartupConfig.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow
+{
+    public class StartupConfig
+    {
+        private readonly List<object> configs = new List<object>();
+
+        public string? NameOverride { get; set; }
+
+        public void Add(object config)
+        {
+            configs.Add(config);
+        }
+
+        public T Get<T>()
+        {
+            var match = configs.Single(c => typeof(T).IsAssignableFrom(c.GetType()));
+            return (T)match;
+        }
+    }
+}

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -0,0 +1,220 @@
+using Logging;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public interface IStartupWorkflow
+    {
+        IKnownLocations GetAvailableLocations();
+        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        CrashWatcher CreateCrashWatcher(RunningContainer container);
+        void Stop(RunningContainers runningContainers);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
+        string ExecuteCommand(RunningContainer container, string command, params string[] args);
+        void DeleteNamespace();
+        void DeleteNamespacesStartingWith(string namespacePrefix);
+    }
+
+    public class StartupWorkflow : IStartupWorkflow
+    {
+        private readonly ILog log;
+        private readonly WorkflowNumberSource numberSource;
+        private readonly K8sCluster cluster;
+        private readonly KnownK8sPods knownK8SPods;
+        private readonly string k8sNamespace;
+        private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
+        private readonly LocationProvider locationProvider;
+
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace)
+        {
+            this.log = log;
+            this.numberSource = numberSource;
+            this.cluster = cluster;
+            this.knownK8SPods = knownK8SPods;
+            this.k8sNamespace = k8sNamespace;
+
+            locationProvider = new LocationProvider(log, K8s);
+        }
+
+        public IKnownLocations GetAvailableLocations()
+        {
+            return locationProvider.GetAvailableLocations();
+        }
+
+        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        {
+            return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
+        }
+
+        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        {
+            return K8s(controller =>
+            {
+                var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
+                var runningPod = controller.BringOnline(recipes, location);
+                var containers = CreateContainers(runningPod, recipes, startupConfig);
+
+                var rc = new RunningContainers(startupConfig, runningPod, containers);
+                cluster.Configuration.Hooks.OnContainersStarted(rc);
+                return rc;
+            });
+        }
+
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return K8s(c => c.CreateCrashWatcher(container));
+        }
+
+        public void Stop(RunningContainers runningContainers)
+        {
+            K8s(controller =>
+            {
+                controller.Stop(runningContainers.RunningPod);
+                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
+            });
+        }
+
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
+        {
+            K8s(controller =>
+            {
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
+            });
+        }
+
+        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
+        {
+            return K8s(controller =>
+            {
+                return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args);
+            });
+        }
+
+        public void DeleteNamespace()
+        {
+            K8s(controller =>
+            {
+                controller.DeleteNamespace();
+            });
+        }
+
+        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        {
+            K8s(controller =>
+            {
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+            });
+        }
+
+        private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
+        {
+            log.Debug();
+            return recipes.Select(r =>
+            {
+                var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
+                log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
+
+                var name = GetContainerName(r, startupConfig);
+
+                return new RunningContainer(runningPod, r, servicePorts, name,
+                    GetContainerExternalAddress(runningPod, servicePorts),
+                    GetContainerInternalAddress(r));
+
+            }).ToArray();
+        }
+
+        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
+        {
+            if (startupConfig == null) return "";
+            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
+            {
+                return $"<{startupConfig.NameOverride}{recipe.Number}>";
+            }
+            else
+            {
+                return $"<{recipe.Name}>";
+            }
+        }
+
+        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        {
+            return new Address(
+                pod.Cluster.HostAddress,
+                GetServicePort(servicePorts));
+        }
+
+        private Address GetContainerInternalAddress(ContainerRecipe recipe)
+        {
+            var serviceName = "service-" + numberSource.WorkflowNumber;
+            var port = GetInternalPort(recipe);
+
+            return new Address(
+                $"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
+                port);
+        }
+
+        private static int GetServicePort(Port[] servicePorts)
+        {
+            if (servicePorts.Any()) return servicePorts.First().Number;
+            return 0;
+        }
+
+        private static int GetInternalPort(ContainerRecipe recipe)
+        {
+            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
+            return 0;
+        }
+
+        private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        {
+            log.Debug();
+            var result = new List<ContainerRecipe>();
+            for (var i = 0; i < numberOfContainers; i++)
+            {
+                var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig);
+                if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName);
+                cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe);
+                result.Add(recipe);
+            }
+
+            return result.ToArray();
+        }
+
+        private void K8s(Action<K8sController> action)
+        {
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
+            action(controller);
+            controller.Dispose();
+        }
+
+        private T K8s<T>(Func<K8sController, T> action)
+        {
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
+            var result = action(controller);
+            controller.Dispose();
+            return result;
+        }
+    }
+
+    public interface ILogHandler
+    {
+        void Log(Stream log);
+    }
+
+    public abstract class LogHandler : ILogHandler
+    {
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                ProcessLine(line);
+                line = reader.ReadLine();
+            }
+        }
+
+        protected abstract void ProcessLine(string line);
+    }
+}

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -0,0 +1,42 @@
+using Logging;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class WorkflowCreator
+    {
+        private readonly NumberSource numberSource = new NumberSource(0);
+        private readonly NumberSource containerNumberSource = new NumberSource(0);
+        private readonly KnownK8sPods knownPods = new KnownK8sPods();
+        private readonly K8sCluster cluster;
+        private readonly ILog log;
+        private readonly Configuration configuration;
+        private readonly string k8sNamespace;
+
+        public WorkflowCreator(ILog log, Configuration configuration)
+        {
+            this.log = log;
+            this.configuration = configuration;
+            cluster = new K8sCluster(configuration);
+            k8sNamespace = configuration.KubernetesNamespace.ToLowerInvariant();
+        }
+
+        public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
+        {
+            var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
+                                                                    containerNumberSource);
+
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, GetNamespace(namespaceOverride));
+        }
+
+        private string GetNamespace(string? namespaceOverride)
+        {
+            if (namespaceOverride != null)
+            {
+                if (!configuration.AllowNamespaceOverride) throw new Exception("Namespace override is not allowed.");
+                return namespaceOverride;
+            }
+            return k8sNamespace;
+        }
+    }
+}

Framework/KubernetesWorkflow/WorkflowNumberSource.cs

@@ -0,0 +1,22 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class WorkflowNumberSource
+    {
+        private readonly NumberSource containerNumberSource;
+
+        public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource)
+        {
+            WorkflowNumber = workflowNumber;
+            this.containerNumberSource = containerNumberSource;
+        }
+
+        public int WorkflowNumber { get; }
+
+        public int GetContainerNumber()
+        {
+            return containerNumberSource.GetNextNumber();
+        }
+    }
+}

Framework/Logging/ApplicationIds.cs

@@ -0,0 +1,20 @@
+namespace Logging
+{
+    public class ApplicationIds
+    {
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
+        {
+            CodexId = codexId;
+            GethId = gethId;
+            PrometheusId = prometheusId;
+            CodexContractsId = codexContractsId;
+            GrafanaId = grafanaId;
+        }
+
+        public string CodexId { get; }
+        public string GethId { get; }
+        public string PrometheusId { get; }
+        public string CodexContractsId { get; }
+        public string GrafanaId { get; }
+    }
+}

Framework/Logging/BaseLog.cs

@@ -0,0 +1,107 @@
+using Utils;
+
+namespace Logging
+{
+    public interface ILog
+    {
+        void Log(string message);
+        void Debug(string message = "", int skipFrames = 0);
+        void Error(string message);
+        void AddStringReplace(string from, string to);
+        LogFile CreateSubfile(string ext = "log");
+    }
+
+    public abstract class BaseLog : ILog
+    {
+        private readonly NumberSource subfileNumberSource = new NumberSource(0);
+        private readonly bool debug;
+        private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
+        private LogFile? logFile;
+
+        protected BaseLog(bool debug)
+        {
+            this.debug = debug;
+        }
+
+        protected abstract string GetFullName();
+
+        public LogFile LogFile 
+        {
+            get
+            {
+                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
+                return logFile;
+            }
+        }
+
+        public virtual void Log(string message)
+        {
+            LogFile.Write(ApplyReplacements(message));
+        }
+
+        public virtual void Debug(string message = "", int skipFrames = 0)
+        {
+            if (debug)
+            {
+                var callerName = DebugStack.GetCallerName(skipFrames);
+                // We don't use Log because in the debug output we should not have any replacements.
+                LogFile.Write($"(debug)({callerName}) {message}");
+            }
+        }
+
+        public virtual void Error(string message)
+        {
+            Log($"[ERROR] {message}");
+        }
+
+        public virtual void AddStringReplace(string from, string to)
+        {
+            if (string.IsNullOrWhiteSpace(from)) return;
+            if (replacements.Any(r => r.From == from)) return;
+            replacements.Add(new BaseLogStringReplacement(from, to));
+        }
+
+        public virtual void Delete()
+        {
+            File.Delete(LogFile.FullFilename);
+        }
+
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+        }
+
+        private string ApplyReplacements(string str)
+        {
+            foreach (var replacement in replacements)
+            {
+                str = replacement.Apply(str);
+            }
+            return str;
+        }
+
+        private string GetSubfileNumber()
+        {
+            return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
+        }
+    }
+
+    public class BaseLogStringReplacement
+    {
+        public BaseLogStringReplacement(string from, string to)
+        {
+            From = from;
+            To = to;
+
+            if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
+        }
+
+        public string From { get; }
+        public string To { get; }
+
+        public string Apply(string msg)
+        {
+            return msg.Replace(From, To);
+        }
+    }
+}

Framework/Logging/ConsoleLog.cs

@@ -0,0 +1,19 @@
+namespace Logging
+{
+    public class ConsoleLog : BaseLog
+    {
+        public ConsoleLog() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

Framework/Logging/LogConfig.cs

@@ -0,0 +1,14 @@
+namespace Logging
+{
+    public class LogConfig
+    {
+        public LogConfig(string logRoot, bool debugEnabled)
+        {
+            LogRoot = logRoot;
+            DebugEnabled = debugEnabled;
+        }
+        
+        public string LogRoot { get; }
+        public bool DebugEnabled { get; }
+    }
+}

Framework/Logging/LogFile.cs

@@ -0,0 +1,61 @@
+namespace Logging
+{
+    public class LogFile
+    {
+        private readonly string extension;
+        private readonly object fileLock = new object();
+        private string filename;
+
+        public LogFile(string filename, string extension)
+        {
+            this.filename = filename;
+            this.extension = extension;
+            FullFilename = filename + "." + extension;
+
+            EnsurePathExists(filename);
+        }
+
+        public string FullFilename { get; private set; }
+
+        public void Write(string message)
+        {
+            WriteRaw($"{GetTimestamp()} {message}");
+        }
+
+        public void WriteRaw(string message)
+        {
+            try
+            {
+                lock (fileLock)
+                { 
+                    File.AppendAllLines(FullFilename, new[] { message });
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
+        }
+
+        public void ConcatToFilename(string toAdd)
+        {
+            var oldFullName = FullFilename;
+
+            filename += toAdd;
+            FullFilename = filename + "." + extension;
+
+            File.Move(oldFullName, FullFilename);
+        }
+
+        private static string GetTimestamp()
+        {
+            return $"[{DateTime.UtcNow.ToString("o")}]";
+        }
+
+        private void EnsurePathExists(string filename)
+        {
+            var path = new FileInfo(filename).Directory!.FullName;
+            Directory.CreateDirectory(path);
+        }
+    }
+}

Framework/Logging/LogPrefixer.cs

@@ -0,0 +1,39 @@
+namespace Logging
+{
+    public class LogPrefixer : ILog
+    {
+        private readonly ILog backingLog;
+        private readonly string prefix;
+
+        public LogPrefixer(ILog backingLog, string prefix)
+        {
+            this.backingLog = backingLog;
+            this.prefix = prefix;
+        }
+
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return backingLog.CreateSubfile(ext);
+        }
+
+        public void Debug(string message = "", int skipFrames = 0)
+        {
+            backingLog.Debug(prefix + message, skipFrames);
+        }
+
+        public void Error(string message)
+        {
+            backingLog.Error(prefix + message);
+        }
+
+        public void Log(string message)
+        {
+            backingLog.Log(prefix + message);
+        }
+
+        public void AddStringReplace(string from, string to)
+        {
+            backingLog.AddStringReplace(from, to);
+        }
+    }
+}

Framework/Logging/LogSplitter.cs

@@ -0,0 +1,42 @@
+namespace Logging
+{
+    public class LogSplitter : ILog
+    {
+        private readonly ILog[] targetLogs;
+
+        public LogSplitter(params ILog[] targetLogs)
+        {
+            this.targetLogs = targetLogs;
+        }
+
+        public void AddStringReplace(string from, string to)
+        {
+            OnAll(l => l.AddStringReplace(from, to));
+        }
+
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return targetLogs.First().CreateSubfile(ext);
+        }
+
+        public void Debug(string message = "", int skipFrames = 0)
+        {
+            OnAll(l => l.Debug(message, skipFrames + 2));
+        }
+
+        public void Error(string message)
+        {
+            OnAll(l => l.Error(message));
+        }
+
+        public void Log(string message)
+        {
+            OnAll(l => l.Log(message));
+        }
+
+        private void OnAll(Action<ILog> action)
+        {
+            foreach (var t in targetLogs) action(t);
+        }
+    }
+}

Framework/Logging/Logging.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+	  <RootNamespace>Logging</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+	  <ProjectReference Include="..\Utils\Utils.csproj" />
+	</ItemGroup>
+</Project>

Framework/Logging/NullLog.cs

@@ -0,0 +1,37 @@
+namespace Logging
+{
+    public class NullLog : BaseLog
+    {
+        public NullLog() : base(false)
+        {
+        }
+
+        public string FullFilename { get; set; } = "NULL";
+
+        protected override string GetFullName()
+        {
+            return FullFilename;
+        }
+
+        public override void Log(string message)
+        {
+        }
+
+        public override void Debug(string message = "", int skipFrames = 0)
+        {
+        }
+
+        public override void Error(string message)
+        {
+            Console.WriteLine("Error: " + message);
+        }
+
+        public override void AddStringReplace(string from, string to)
+        {
+        }
+
+        public override void Delete()
+        {
+        }
+    }
+}

Framework/Logging/Stopwatch.cs

@@ -0,0 +1,71 @@
+using Utils;
+
+namespace Logging
+{
+    public class Stopwatch
+    {
+        private readonly DateTime start = DateTime.UtcNow;
+        private readonly ILog log;
+        private readonly string name;
+        private readonly bool debug;
+
+        private Stopwatch(ILog log, string name, bool debug)
+        {
+            this.log = log;
+            this.name = name;
+            this.debug = debug;
+        }
+
+        public static void Measure(ILog log, string name, Action action, bool debug = false)
+        {
+            var sw = Begin(log, name, debug);
+            action();
+            sw.End();
+        }
+
+        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
+        {
+            var sw = Begin(log, name, debug);
+            var result = action();
+            sw.End();
+            return result;
+        }
+
+        public static Stopwatch Begin(ILog log)
+        {
+            return Begin(log, "");
+        }
+
+        public static Stopwatch Begin(ILog log, string name)
+        {
+            return Begin(log, name, false);
+        }
+
+        public static Stopwatch Begin(ILog log, bool debug)
+        {
+            return Begin(log, "", debug);
+        }
+
+        public static Stopwatch Begin(ILog log, string name, bool debug)
+        {
+            return new Stopwatch(log, name, debug);
+        }
+
+        public TimeSpan End(string msg = "", int skipFrames = 0)
+        {
+            var duration = DateTime.UtcNow - start;
+            var entry = $"{name} {msg} ({Time.FormatDuration(duration)})";
+
+            if (debug)
+            {
+                log.Debug(entry, 1 + skipFrames);
+            }
+            else
+            {
+                log.Log(entry);
+            }
+
+            return duration;
+        }
+    }
+}

Framework/NethereumWorkflow/ConversionExtensions.cs

@@ -0,0 +1,30 @@
+using Nethereum.Hex.HexTypes;
+using System.Numerics;
+
+namespace NethereumWorkflow
+{
+    public static class ConversionExtensions
+    {
+        public static HexBigInteger ToHexBig(this decimal amount)
+        {
+            var bigint = ToBig(amount);
+            var str = bigint.ToString("X");
+            return new HexBigInteger(str);
+        }
+
+        public static BigInteger ToBig(this decimal amount)
+        {
+            return new BigInteger(amount);
+        }
+
+        public static decimal ToDecimal(this HexBigInteger hexBigInteger)
+        {
+            return ToDecimal(hexBigInteger.Value);
+        }
+
+        public static decimal ToDecimal(this BigInteger bigInteger)
+        {
+            return (decimal)bigInteger;
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -0,0 +1,74 @@
+using Logging;
+using Nethereum.Contracts;
+using Nethereum.RPC.Eth.DTOs;
+using Nethereum.Web3;
+using Utils;
+
+namespace NethereumWorkflow
+{
+    public class NethereumInteraction
+    {
+        private readonly ILog log;
+        private readonly Web3 web3;
+
+        internal NethereumInteraction(ILog log, Web3 web3)
+        {
+            this.log = log;
+            this.web3 = web3;
+        }
+
+        public void SendEth(string toAddress, decimal ethAmount)
+        {
+            var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
+            if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
+        }
+
+        public decimal GetEthBalance()
+        {
+            return GetEthBalance(web3.TransactionManager.Account.Address);
+        }
+
+        public decimal GetEthBalance(string address)
+        {
+            var balance = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(address));
+            return Web3.Convert.FromWei(balance.Value);
+        }
+
+        public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        {
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
+        }
+
+        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        {
+            var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
+            var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
+            if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");
+        }
+
+        public decimal? GetSyncedBlockNumber()
+        {
+            log.Debug();
+            var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            var numberOfBlocks = number.ToDecimal();
+            if (sync.IsSyncing) return null;
+            return numberOfBlocks;
+        }
+
+        public bool IsContractAvailable(string abi, string contractAddress)
+        {
+            log.Debug();
+            try
+            {
+                var contract = web3.Eth.GetContract(abi, contractAddress);
+                return contract != null;
+            }
+            catch
+            {
+                return false;
+            }
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -0,0 +1,32 @@
+using Logging;
+using Nethereum.Web3;
+
+namespace NethereumWorkflow
+{
+    public class NethereumInteractionCreator
+    {
+        private readonly ILog log;
+        private readonly string ip;
+        private readonly int port;
+        private readonly string privateKey;
+
+        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        {
+            this.log = log;
+            this.ip = ip;
+            this.port = port;
+            this.privateKey = privateKey;
+        }
+
+        public NethereumInteraction CreateWorkflow()
+        {
+            return new NethereumInteraction(log, CreateWeb3());
+        }
+
+        private Web3 CreateWeb3()
+        {
+            var account = new Nethereum.Web3.Accounts.Account(privateKey);
+            return new Web3(account, $"{ip}:{port}");
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+	<RootNamespace>NethereumWorkflow</RootNamespace>
+	<ImplicitUsings>enable</ImplicitUsings>
+	<Nullable>enable</Nullable>
+
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/Utils/Address.cs

@@ -0,0 +1,14 @@
+namespace Utils
+{
+    public class Address
+    {
+        public Address(string host, int port)
+        {
+            Host = host;
+            Port = port;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -0,0 +1,104 @@
+namespace Utils
+{
+    public class ByteSize
+    {
+        public ByteSize(long sizeInBytes)
+        {
+            if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
+            SizeInBytes = sizeInBytes;
+        }
+
+        public long SizeInBytes { get; }
+
+        public const double DefaultSecondsPerMB = 10.0;
+
+        public long ToMB()
+        {
+            return SizeInBytes / (1024 * 1024);
+        }
+
+        public ByteSize Multiply(double factor)
+        {
+            double bytes = SizeInBytes;
+            double result = Math.Round(bytes * factor);
+            return new ByteSize(Convert.ToInt64(result));
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ByteSize size && SizeInBytes == size.SizeInBytes;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(SizeInBytes);
+        }
+
+        public override string ToString()
+        {
+            return Formatter.FormatByteSize(SizeInBytes);
+        }
+
+        public TimeSpan ToTimeSpan(double secsPerMB = DefaultSecondsPerMB)
+        {
+            var filesizeInMb = SizeInBytes / (1024 * 1024);
+            return TimeSpan.FromSeconds(filesizeInMb * secsPerMB);
+        }
+    }
+
+    public static class ByteSizeIntExtensions
+    {
+        private const long Kilo = 1024;
+
+        public static ByteSize Bytes(this long i)
+        {
+            return new ByteSize(i);
+        }
+
+        public static ByteSize KB(this long i)
+        {
+            return new ByteSize(i * Kilo);
+        }
+
+        public static ByteSize MB(this long i)
+        {
+            return (i * Kilo).KB();
+        }
+
+        public static ByteSize GB(this long i)
+        {
+            return (i * Kilo).MB();
+        }
+
+        public static ByteSize TB(this long i)
+        {
+            return (i * Kilo).GB();
+        }
+
+        public static ByteSize Bytes(this int i)
+        {
+            return new ByteSize(i);
+        }
+
+        public static ByteSize KB(this int i)
+        {
+            return Convert.ToInt64(i).KB();
+        }
+
+        public static ByteSize MB(this int i)
+        {
+            return Convert.ToInt64(i).MB();
+        }
+
+        public static ByteSize GB(this int i)
+        {
+            return Convert.ToInt64(i).GB();
+        }
+
+        public static ByteSize TB(this int i)
+        {
+            return Convert.ToInt64(i).TB();
+        }
+
+    }
+}

Framework/Utils/DebugStack.cs

@@ -0,0 +1,12 @@
+using System.Diagnostics;
+
+namespace Utils
+{
+    public class DebugStack
+    {
+        public static string GetCallerName(int skipFrames = 0)
+        {
+            return new StackFrame(2 + skipFrames, true).GetMethod()!.Name;
+        }
+    }
+}

Framework/Utils/Formatter.cs

@@ -0,0 +1,16 @@
+namespace Utils
+{
+    public static class Formatter
+    {
+        private static readonly string[] sizeSuffixes = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
+
+        public static string FormatByteSize(long bytes)
+        {
+            if (bytes == 0) return "0" + sizeSuffixes[0];
+
+            var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
+            var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
+            return digit.ToString() + sizeSuffixes[sizeOrder];
+        }
+    }
+}

Framework/Utils/FrameworkAssert.cs

@@ -0,0 +1,15 @@
+namespace Utils
+{
+    public static class FrameworkAssert
+    {
+        public static void That(bool condition, string message)
+        {
+            if (!condition) Fail(message);
+        }
+
+        public static void Fail(string message)
+        {
+            throw new Exception(message);
+        }
+    }
+}

Framework/Utils/NumberSource.cs

@@ -1,4 +1,4 @@
-namespace CodexDistTestCore
+namespace Utils
 {
     public class NumberSource
     {

Framework/Utils/ParseEnum.cs

@@ -0,0 +1,10 @@
+namespace Utils
+{
+    public static class ParseEnum
+    {
+        public static T Parse<T>(string value)
+        {
+            return (T)Enum.Parse(typeof(T), value, true);
+        }
+    }
+}