Fixes too-frequent save of overview json. Resets failure count on upload.

# Conflicts:
#	Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

.github/workflows/docker-autoclient.yml

@@ -1,5 +1,6 @@
 name: Docker - AutoClient
 
+
 on:
   push:
     branches:
@@ -12,15 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-autoclient.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
+
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/AutoClient/docker/Dockerfile
-      docker_repo: codexstorage/codex-autoclient
+      dockerhub_repo: codexstorage/codex-autoclient
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-discordbot.yml

@@ -13,16 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-discordbot.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/BiblioTech/docker/Dockerfile
-      docker_repo: codexstorage/codex-discordbot
+      dockerhub_repo: codexstorage/codex-discordbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-keymaker.yml

@@ -11,17 +11,16 @@ on:
       - 'Tools/KeyMaker/**'
       - 'Framework/**'
       - 'ProjectPlugins/**'
-      - .github/workflows/docker-KeyMaker.yml
-      - .github/workflows/docker-reusable.yml
+      - .github/workflows/docker-keymaker.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/KeyMaker/docker/Dockerfile
-      docker_repo: codexstorage/codex-keymaker
+      dockerhub_repo: codexstorage/codex-keymaker
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-marketinsights.yml

@@ -12,16 +12,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-marketinsights.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/MarketInsights/Dockerfile
-      docker_repo: codexstorage/codex-marketinsights
+      dockerhub_repo: codexstorage/codex-marketinsights
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-reusable.yml

@@ -1,178 +0,0 @@
-name: Reusable - Docker
-
-
-on:
-  workflow_call:
-    inputs:
-      docker_file:
-        default: docker/Dockerfile
-        description: Dockerfile
-        required: false
-        type: string
-      docker_repo:
-        default: codexstorage/cs-codex-dist-tests
-        description: DockerHub repository
-        required: false
-        type: string
-      tag_latest:
-        default: true
-        description: Set latest tag for Docker images
-        required: false
-        type: boolean
-      tag_sha:
-        default: true
-        description: Set Git short commit as Docker tag
-        required: false
-        type: boolean
-      tag_suffix:
-        default: ''
-        description: Suffix for Docker images tag
-        required: false
-        type: string
-
-
-env:
-  DOCKER_FILE: ${{ inputs.docker_file }}
-  DOCKER_REPO: ${{ inputs.docker_repo }}
-  TAG_LATEST: ${{ inputs.tag_latest }}
-  TAG_SHA: ${{ inputs.tag_sha }}
-  TAG_SUFFIX: ${{ inputs.tag_suffix }}
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
-
-      - name: Docker - Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Docker - Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.target.arch }}
-          path: /tmp/digests
-          if-no-files-found: error
-          retention-days: 1
-
-
-  # Publish multi-platform image
-  publish:
-    name: Publish multi-platform image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Variables
-        run: |
-          # Adjust custom suffix when set and
-          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
-            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-          fi
-          # Disable SHA tags on tagged release
-          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
-            echo "TAG_SHA=false" >>$GITHUB_ENV
-          fi
-          # Handle latest and latest-custom using raw
-          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
-            echo "TAG_LATEST=false" >>$GITHUB_ENV
-            echo "TAG_RAW=true" >>$GITHUB_ENV
-            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
-              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
-            else
-              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-            fi
-          else
-            echo "TAG_RAW=false" >>$GITHUB_ENV
-          fi
-
-      - name: Docker - Download digests
-        uses: actions/download-artifact@v4
-        with:
-          pattern: digests-*
-          merge-multiple: true
-          path: /tmp/digests
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=${{ env.TAG_LATEST }}
-            suffix=${{ env.TAG_SUFFIX }},onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=raw,enable=${{ env.TAG_RAW }},value=latest
-            type=sha,enable=${{ env.TAG_SHA }}
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
-
-      - name: Docker - Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker-rewarder.yml

@@ -1,5 +1,6 @@
 name: Docker - Rewarder Bot
 
+
 on:
   push:
     branches:
@@ -12,15 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-rewarder.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
+
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/TestNetRewarder/docker/Dockerfile
-      docker_repo: codexstorage/codex-rewarderbot
+      dockerhub_repo: codexstorage/codex-rewarderbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-runner.yml

@@ -11,12 +11,15 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker-runner.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: docker/Dockerfile
+      dockerhub_repo: codexstorage/cs-codex-dist-tests
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit

.github/workflows/run-release-tests.yaml

@@ -109,6 +109,7 @@ jobs:
           # Get logs
           while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
             echo "Show ${pod} logs ..."
+            echo "----"
             kubectl logs $pod -n $namespace -f || true
             sleep 1
           done

.gitignore

@@ -3,3 +3,4 @@ obj
 bin
 .vscode
 Tools/AutoClient/datapath
+.editorconfig

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/Core/Core.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
     <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\WebUtils\WebUtils.csproj" />
   </ItemGroup>
 
 </Project>

Framework/Core/CoreInterface.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;
 
 namespace Core
 {

Framework/Core/EntryPoint.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -8,16 +9,16 @@ namespace Core
         private readonly IToolsFactory toolsFactory;
         private readonly PluginManager manager = new PluginManager();
 
-        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
-            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
 
             Tools = toolsFactory.CreateTools();
             manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
         }
 
         public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
-            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+            : this(log, configuration, fileManagerRootFolder, new DefaultWebCallTimeSet(), new DefaultK8sTimeSet())
         {
         }
 

Framework/Core/PluginTools.cs

@@ -1,12 +1,14 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
-    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
     {
-        ITimeSet TimeSet { get; }
+        IWebCallTimeSet WebCallTimeSet { get; }
+        IK8sTimeSet K8STimeSet { get; }
 
         /// <summary>
         /// Deletes kubernetes and tracked file resources.
@@ -25,13 +27,6 @@ namespace Core
         ILog GetLog();
     }
 
-    public interface IHttpFactoryTool
-    {
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp(string id);
-    }
-
     public interface IFileTool
     {
         IFileManager GetFileManager();
@@ -40,18 +35,22 @@ namespace Core
     internal class PluginTools : IPluginTools
     {
         private readonly WorkflowCreator workflowCreator;
+        private readonly HttpFactory httpFactory;
         private readonly IFileManager fileManager;
         private readonly LogPrefixer log;
 
-        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            TimeSet = timeSet;
+            httpFactory = new HttpFactory(log, webCallTimeSet);
+            WebCallTimeSet = webCallTimeSet;
+            K8STimeSet = k8STimeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
-        public ITimeSet TimeSet { get; }
+        public IWebCallTimeSet WebCallTimeSet { get; }
+        public IK8sTimeSet K8STimeSet { get; }
 
         public void ApplyLogPrefix(string prefix)
         {
@@ -60,17 +59,17 @@ namespace Core
 
         public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(id, onClientCreated, TimeSet);
+            return httpFactory.CreateHttp(id, onClientCreated);
         }
 
-        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet)
         {
-            return new Http(id, log, ts, onClientCreated);
+            return httpFactory.CreateHttp(id, onClientCreated, timeSet);
         }
 
         public IHttp CreateHttp(string id)
         {
-            return new Http(id, log, TimeSet);
+            return httpFactory.CreateHttp(id);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/Core/ToolsFactory.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -13,19 +14,21 @@ namespace Core
         private readonly ILog log;
         private readonly WorkflowCreator workflowCreator;
         private readonly string fileManagerRootFolder;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet webCallTimeSet;
+        private readonly IK8sTimeSet k8STimeSet;
 
-        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = log;
             workflowCreator = new WorkflowCreator(log, configuration);
             this.fileManagerRootFolder = fileManagerRootFolder;
-            this.timeSet = timeSet;
+            this.webCallTimeSet = webCallTimeSet;
+            this.k8STimeSet = k8STimeSet;
         }
 
         public PluginTools CreateTools()
         {
-            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
         }
     }
 }

Framework/FileUtils/TrackedFile.cs

@@ -14,6 +14,12 @@ namespace FileUtils
             Label = label;
         }
 
+        public static TrackedFile FromPath(ILog log, string filepath)
+        {
+            // todo: I don't wanne have to do this to call upload.
+            return new TrackedFile(log, filepath, string.Empty);
+        }
+
         public string Filename { get; }
         public string Label { get; }
 

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
@@ -19,7 +20,7 @@ namespace GethConnector
                 return null;
             }
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
 
             var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
 

Framework/GethConnector/GethInput.cs

@@ -45,8 +45,12 @@
         private static string? GetEnvVar(List<string> error, string name)
         {
             var result = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
-            return result;
+            if (string.IsNullOrEmpty(result))
+            {
+                error.Add($"'{name}' is not set.");
+                return null;
+            }
+            return result.Trim();
         }
     }
 }

Framework/KubernetesWorkflow/ContainerCrashWatcher.cs

@@ -3,7 +3,7 @@ using Logging;
 
 namespace KubernetesWorkflow
 {
-    public class CrashWatcher
+    public class ContainerCrashWatcher
     {
         private readonly ILog log;
         private readonly KubernetesClientConfiguration config;
@@ -15,7 +15,7 @@ namespace KubernetesWorkflow
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
@@ -45,7 +45,7 @@ namespace KubernetesWorkflow
             if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
         }
 
-        public bool HasContainerCrashed()
+        public bool HasCrashed()
         {
             using var client = new Kubernetes(config);
             var result = HasContainerBeenRestarted(client);
@@ -70,13 +70,13 @@ namespace KubernetesWorkflow
             using var client = new Kubernetes(config);
             while (!token.IsCancellationRequested)
             {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
-
                 if (HasContainerBeenRestarted(client))
                 {
                     DownloadCrashedContainerLogs(client);
                     return;
                 }
+
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
             }
         }
 
@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
             handler.Log(stream);
         }
     }

Framework/KubernetesWorkflow/K8sController.cs

@@ -122,6 +122,16 @@ namespace KubernetesWorkflow
             var all = client.Run(c => c.ListNamespace().Items);
             var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
 
+            if (wait)
+            {
+                // If we're going to wait, trigger the delete for all the namespaces immediately.
+                // Then wait for them to finish one by one.
+                foreach (var ns in namespaces)
+                {
+                    DeleteNamespace(ns, false);
+                }
+            }
+
             foreach (var ns in namespaces)
             {
                 DeleteNamespace(ns, wait);
@@ -735,7 +745,10 @@ namespace KubernetesWorkflow
                 throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
                     $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
             }
-            return pods[0];
+            var pod = pods[0];
+            if (pod.Status == null) throw new Exception("Pod status unknown");
+            if (string.IsNullOrEmpty(pod.Status.PodIP)) throw new Exception("Pod IP unknown");
+            return pod;
         }
 
         #endregion
@@ -906,7 +919,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }
@@ -946,13 +959,13 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             var containerName = container.Name;
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new ContainerCrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/K8sTimeSet.cs

@@ -0,0 +1,42 @@
+namespace Core
+{
+    public interface IK8sTimeSet
+    {
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}

Framework/KubernetesWorkflow/LogHandler.cs

@@ -25,11 +25,11 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
-            var msg = $"{description} -->> {LogFile.FullFilename}";
+            var msg = $"{description} -->> {LogFile.Filename}";
             sourceLog.Log(msg);
 
             LogFile.Write(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
         FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
         PodInfo GetPodInfo(RunningPod pod);
-        CrashWatcher CreateCrashWatcher(RunningContainer container);
+        ContainerCrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
         IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
@@ -61,7 +61,8 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
+                var info = GetPodInfo(startResult.Deployment);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), info, startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
@@ -83,9 +84,14 @@ namespace KubernetesWorkflow
             });
         }
 
+        public PodInfo GetPodInfo(RunningDeployment deployment)
+        {
+            return K8s(c => c.GetPodInfo(deployment));
+        }
+
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
+            return GetPodInfo(container.RunningPod.StartResult.Deployment);
         }
 
         public PodInfo GetPodInfo(RunningPod pod)
@@ -93,7 +99,7 @@ namespace KubernetesWorkflow
             return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             return K8s(c => c.CreateCrashWatcher(container));
         }
@@ -127,14 +133,14 @@ namespace KubernetesWorkflow
         {
             var msg = $"Downloading container log for '{container.Name}'";
             log.Log(msg);
-            var logHandler = new WriteToFileLogHandler(log, msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
 
             K8s(controller =>
             {
                 controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
 
-            return new DownloadedLog(logHandler, container.Name);
+            return new DownloadedLog(logHandler.LogFile, container.Name);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -209,6 +215,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetExternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{recipe.Name}:{tag}",
                 startResult.Cluster.HostAddress,
                 port.Number);
         }
@@ -220,6 +227,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetInternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{serviceName}:{tag}",
                 $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                 port.Number);
         }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -4,9 +4,10 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningPod
     {
-        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, PodInfo podInfo, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             Id = id;
+            PodInfo = podInfo;
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
@@ -15,6 +16,7 @@ namespace KubernetesWorkflow.Types
         }
 
         public string Id { get; }
+        public PodInfo PodInfo { get; }
         public StartupConfig StartupConfig { get; }
         public StartResult StartResult { get; }
         public RunningContainer[] Containers { get; }

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -34,7 +34,7 @@ namespace Logging
         {
             get
             {
-                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
+                if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
                 return logFile;
             }
         }
@@ -69,12 +69,16 @@ namespace Logging
 
         public virtual void Delete()
         {
-            File.Delete(LogFile.FullFilename);
+            File.Delete(LogFile.Filename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/DownloadedLog.cs

@@ -1,10 +1,8 @@
-using Logging;
-
-namespace KubernetesWorkflow
+namespace Logging
 {
     public interface IDownloadedLog
     {
-        string ContainerName { get; }
+        string SourceName { get; }
 
         void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
@@ -14,21 +12,27 @@ namespace KubernetesWorkflow
         void DeleteFile();
     }
 
-    internal class DownloadedLog : IDownloadedLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
+        public DownloadedLog(string filepath, string sourceName)
         {
-            logFile = logHandler.LogFile;
-            ContainerName = containerName;
+            logFile = new LogFile(filepath);
+            SourceName = sourceName;
         }
 
-        public string ContainerName { get; }
+        public DownloadedLog(LogFile logFile, string sourceName)
+        {
+            this.logFile = logFile;
+            SourceName = sourceName;
+        }
+
+        public string SourceName { get; }
 
         public void IterateLines(Action<string> action)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
+            using var file = File.OpenRead(logFile.Filename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
@@ -64,12 +68,12 @@ namespace KubernetesWorkflow
 
         public string GetFilepath()
         {
-            return logFile.FullFilename;
+            return logFile.Filename;
         }
 
         public void DeleteFile()
         {
-            File.Delete(logFile.FullFilename);
+            File.Delete(logFile.Filename);
         }
     }
 }

Framework/Logging/LogFile.cs

@@ -1,21 +1,19 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
-        private readonly string extension;
         private readonly object fileLock = new object();
-        private string filename;
 
-        public LogFile(string filename, string extension)
+        public LogFile(string filename)
         {
-            this.filename = filename;
-            this.extension = extension;
-            FullFilename = filename + "." + extension;
+            Filename = filename;
 
             EnsurePathExists(filename);
         }
 
-        public string FullFilename { get; private set; }
+        public string Filename { get; private set; }
 
         public void Write(string message)
         {
@@ -28,7 +26,7 @@
             {
                 lock (fileLock)
                 { 
-                    File.AppendAllLines(FullFilename, new[] { message });
+                    File.AppendAllLines(Filename, new[] { message });
                 }
             }
             catch (Exception ex)
@@ -37,19 +35,24 @@
             }
         }
 
-        public void ConcatToFilename(string toAdd)
+        public void WriteRawMany(IEnumerable<string> lines)
         {
-            var oldFullName = FullFilename;
-
-            filename += toAdd;
-            FullFilename = filename + "." + extension;
-
-            File.Move(oldFullName, FullFilename);
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(Filename, lines);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
         }
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)
@@ -50,6 +50,21 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
+        public TResult Call<TFunction, TResult>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function, new BlockParameter(blockNumber)));
+        }
+
+        public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber)));
+            var aaaa = 0;
+        }
+
         public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
             log.Debug();

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/Utils/Address.cs

@@ -2,12 +2,14 @@
 {
     public class Address
     {
-        public Address(string host, int port)
+        public Address(string logName, string host, int port)
         {
+            LogName = logName;
             Host = host;
             Port = port;
         }
 
+        public string LogName { get; }
         public string Host { get; }
         public int Port { get; }
 
@@ -20,5 +22,21 @@
         {
             return !string.IsNullOrEmpty(Host) && Port > 0;
         }
+
+        public static Address Empty()
+        {
+            return new Address(string.Empty, string.Empty, 0);
+        }
     }
+
+    public interface IHasMetricsScrapeTarget
+    {
+        Address GetMetricsScrapeTarget();
+    }
+    
+    public interface IHasManyMetricScrapeTargets
+    {
+        Address[] GetMetricsScrapeTargets();
+    }
+
 }

Framework/Utils/Base58.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Utils
+{
+    public static class Base58
+    {
+        private const string Digits = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
+
+        public static string Encode(byte[] data)
+        {
+            // Decode byte[] to BigInteger
+            BigInteger intData = 0;
+            for (int i = 0; i < data.Length; i++)
+            {
+                intData = intData * 256 + data[i];
+            }
+
+            // Encode BigInteger to Base58 string
+            string result = "";
+            while (intData > 0)
+            {
+                int remainder = (int)(intData % 58);
+                intData /= 58;
+                result = Digits[remainder] + result;
+            }
+
+            // Append `1` for each leading 0 byte
+            for (int i = 0; i < data.Length && data[i] == 0; i++)
+            {
+                result = '1' + result;
+            }
+            return result;
+        }
+
+        public static byte[] Decode(string s)
+        {
+            BigInteger intData = 0;
+            for (int i = 0; i < s.Length; i++)
+            {
+                int digit = Digits.IndexOf(s[i]); //Slow
+                if (digit < 0)
+                    throw new FormatException(string.Format("Invalid Base58 character `{0}` at position {1}", s[i], i));
+                intData = intData * 58 + digit;
+            }
+
+            // Encode BigInteger to byte[]
+            // Leading zero bytes get encoded as leading `1` characters
+            int leadingZeroCount = s.TakeWhile(c => c == '1').Count();
+            var leadingZeros = Enumerable.Repeat((byte)0, leadingZeroCount);
+            var bytesWithoutLeadingZeros =
+                intData.ToByteArray()
+                .Reverse()// to big endian
+                .SkipWhile(b => b == 0);//strip sign byte
+            var result = leadingZeros.Concat(bytesWithoutLeadingZeros).ToArray();
+            return result;
+        }
+    }
+}

Framework/Utils/CrashWatcher.cs

@@ -0,0 +1,9 @@
+namespace Utils
+{
+    //public interface ICrashWatcher
+    //{
+    //    void Start();
+    //    void Stop();
+    //    bool HasCrashed();
+    //}
+}

Framework/Utils/EthAccount.cs

@@ -0,0 +1,20 @@
+namespace Utils
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
+    }
+}

Framework/Utils/EthAddress.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public interface IHasEthAddress
     {

Framework/Utils/EthTokenExtensions.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public class Ether : IComparable<Ether>
     {

Framework/Utils/Retry.cs

@@ -98,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/TestTokenExtensions.cs

@@ -1,6 +1,6 @@
 using System.Numerics;
 
-namespace CodexContractsPlugin
+namespace Utils
 {
     public class TestToken : IComparable<TestToken>
     {
@@ -81,7 +81,7 @@ namespace CodexContractsPlugin
     {
         public static TestToken TstWei(this int i)
         {
-            return TstWei(Convert.ToDecimal(i));
+            return new TestToken(new BigInteger(i));
         }
 
         public static TestToken TstWei(this decimal i)
@@ -94,6 +94,11 @@ namespace CodexContractsPlugin
             return new TestToken(i);
         }
 
+        public static TestToken TstWei(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s));
+        }
+
         public static TestToken Tst(this int i)
         {
             return Tst(Convert.ToDecimal(i));
@@ -108,5 +113,10 @@ namespace CodexContractsPlugin
         {
             return new TestToken(i * TestToken.WeiFactor);
         }
+
+        public static TestToken Tst(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s) * TestToken.WeiFactor);
+        }
     }
 }

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

Framework/WebUtils/Endpoint.cs

@@ -1,11 +1,10 @@
-using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using Logging;
+using Newtonsoft.Json;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IEndpoint
     {
@@ -119,7 +118,7 @@ namespace Core
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
             {
-                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                Error = delegate (object? sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
                 {
                     if (args.CurrentObject == args.ErrorContext.OriginalObject)
                     {

Framework/WebUtils/Http.cs

@@ -1,7 +1,7 @@
 using Logging;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IHttp
     {
@@ -16,16 +16,16 @@ namespace Core
         private static object lockLock = new object();
         private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet timeSet;
         private readonly Action<HttpClient> onClientCreated;
         private readonly string id;
 
-        internal Http(string id, ILog log, ITimeSet timeSet)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet)
             : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.id = id;
             this.log = log;

Framework/WebUtils/HttpFactory.cs

@@ -0,0 +1,43 @@
+using Logging;
+
+namespace WebUtils
+{
+    public interface IHttpFactory
+    {
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
+        IHttp CreateHttp(string id);
+    }
+
+    public class HttpFactory : IHttpFactory
+    {
+        private readonly ILog log;
+        private readonly IWebCallTimeSet defaultTimeSet;
+
+        public HttpFactory(ILog log)
+            : this (log, new DefaultWebCallTimeSet())
+        {
+        }
+
+        public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet)
+        {
+            this.log = log;
+            this.defaultTimeSet = defaultTimeSet;
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(id, onClientCreated, defaultTimeSet);
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
+        {
+            return new Http(id, log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp(string id)
+        {
+            return new Http(id, log, defaultTimeSet);
+        }
+    }
+}

Framework/WebUtils/WebCallTimeSet.cs

@@ -1,6 +1,6 @@
-namespace Core
+namespace WebUtils
 {
-    public interface ITimeSet
+    public interface IWebCallTimeSet
     {
         /// <summary>
         /// Timeout for a single HTTP call.
@@ -17,20 +17,9 @@
         /// After a failed HTTP call, wait this long before trying again.
         /// </summary>
         TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan K8sOperationTimeout();
     }
 
-    public class DefaultTimeSet : ITimeSet
+    public class DefaultWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -46,19 +35,9 @@
         {
             return TimeSpan.FromSeconds(1);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(30);
-        }
     }
 
-    public class LongTimeSet : ITimeSet
+    public class LongWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -74,15 +53,5 @@
         {
             return TimeSpan.FromSeconds(20);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromHours(1);
-        }
     }
 }

Framework/WebUtils/WebUtils.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	</ItemGroup>
+
+	<ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -1,37 +1,72 @@
 using CodexOpenApi;
-using Core;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
+using WebUtils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class CodexAccess
     {
         private readonly ILog log;
-        private readonly IPluginTools tools;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControl processControl;
+        private ICodexInstance instance;
         private readonly Mapper mapper = new Mapper();
 
-        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
+        public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
         {
-            this.tools = tools;
-            log = tools.GetLog();
-            Container = container;
-            CrashWatcher = crashWatcher;
-
-            CrashWatcher.Start();
+            this.log = log;
+            this.httpFactory = httpFactory;
+            this.processControl = processControl;
+            this.instance = instance;
         }
 
-        public RunningPod Container { get; }
-        public CrashWatcher CrashWatcher { get; }
+        public void Stop(bool waitTillStopped)
+        {
+            processControl.Stop(waitTillStopped);
+            // Prevents accidental use after stop:
+            instance = null!;
+        }
+
+        public IDownloadedLog DownloadLog(string additionalName = "")
+        {
+            var file = log.CreateSubfile(GetName() + additionalName);
+            Log($"Downloading logs to '{file.Filename}'");
+            return processControl.DownloadLog(file);
+        }
+
+        public string GetImageName()
+        {
+            return instance.ImageName;
+        }
+
+        public DateTime GetStartUtc()
+        {
+            return instance.StartUtc;
+        }
 
         public DebugInfo GetDebugInfo()
         {
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public void SetLogLevel(string logLevel)
+        {
+            try
+            {
+                OnCodex(async api =>
+                {
+                    await api.SetDebugLogLevelAsync(logLevel);
+                    return string.Empty;
+                });
+            }
+            catch (Exception exc)
+            {
+                log.Error("Failed to set log level: " + exc);
+            }
+        }
+
         public string GetSpr()
         {
             return CrashCheck(() =>
@@ -79,19 +114,14 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput)
         {
-            return OnCodex(
-                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
-                CreateRetryConfig(nameof(UploadFile), onFailure));
+            return OnCodex(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream));
         }
 
-        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
+        public Stream DownloadFile(string contentId)
         {
-            var fileResponse = OnCodex(
-                api => api.DownloadNetworkStreamAsync(contentId),
-                CreateRetryConfig(nameof(DownloadFile), onFailure));
-
+            var fileResponse = OnCodex(api => api.DownloadNetworkStreamAsync(contentId));
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
@@ -138,17 +168,25 @@ namespace CodexPlugin
             return mapper.Map(space);
         }
 
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        public StoragePurchase? GetPurchaseStatus(string purchaseId)
         {
             return CrashCheck(() =>
             {
                 var endpoint = GetEndpoint();
-                return Time.Retry(() =>
+                try
                 {
-                    var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
-                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                    return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                }, nameof(GetPurchaseStatus));
+                    return Time.Retry(() =>
+                    {
+                        var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                        if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                        return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+                    }, nameof(GetPurchaseStatus));
+                }
+                catch (Exception exc)
+                {
+                    log.Error($"Failed to fetch purchase information for id: '{purchaseId}'. Exception: {exc.Message}");
+                    return null;
+                }
             });
 
             // TODO: current getpurchase api does not line up with its openapi spec.
@@ -157,47 +195,60 @@ namespace CodexPlugin
 
         public string GetName()
         {
-            return Container.Name;
+            return instance.Name;
         }
 
-        public PodInfo GetPodInfo()
+        public Address GetDiscoveryEndpoint()
         {
-            var workflow = tools.CreateWorkflow();
-            return workflow.GetPodInfo(Container);
+            return instance.DiscoveryEndpoint;
         }
 
-        public void DeleteRepoFolder()
+        public Address GetApiEndpoint()
         {
-            try
-            {
-                var containerNumber = Container.Containers.First().Recipe.Number;
-                var dataDir = $"datadir{containerNumber}";
-                var workflow = tools.CreateWorkflow();
-                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
-                Log("Deleted repo folder.");
-            }
-            catch (Exception e)
-            {
-                Log("Unable to delete repo folder: " + e);
-            }
+            return instance.ApiEndpoint;
         }
 
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
+        public Address GetListenEndpoint()
         {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
+            return instance.ListenEndpoint;
+        }
+
+        public bool HasCrashed()
+        {
+            return processControl.HasCrashed();
+        }
+
+        public Address? GetMetricsEndpoint()
+        {
+            return instance.MetricsEndpoint;
+        }
+
+        public EthAccount? GetEthAccount()
+        {
+            return instance.EthAccount;
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            processControl.DeleteDataDirFolder();
+        }
+
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
+        {
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
             return result;
         }
 
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action, Retry retry)
         {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), retry);
             return result;
         }
 
-        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
+        private T CallCodex<T>(HttpClient client, Func<CodexApiClient, Task<T>> action)
         {
             var address = GetAddress();
-            var api = new CodexApi(client);
+            var api = new CodexApiClient(client);
             api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
             return CrashCheck(() => Time.Wait(action(api)));
         }
@@ -210,20 +261,20 @@ namespace CodexPlugin
             }
             finally
             {
-                CrashWatcher.HasContainerCrashed();
+                CheckContainerCrashed();
             }
         }
 
         private IEndpoint GetEndpoint()
         {
-            return tools
-                .CreateHttp(GetHttpId(), CheckContainerCrashed)
-                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
+            return httpFactory
+                .CreateHttp(GetHttpId(), h => CheckContainerCrashed())
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", GetName());
         }
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
+            return instance.ApiEndpoint;
         }
 
         private string GetHttpId()
@@ -231,52 +282,9 @@ namespace CodexPlugin
             return GetAddress().ToString();
         }
 
-        private void CheckContainerCrashed(HttpClient client)
+        private void CheckContainerCrashed()
         {
-            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
-        }
-
-        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
-        {
-            var timeSet = tools.TimeSet;
-
-            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
-            {
-                onFailure(failure);
-                Investigate(failure, timeSet);
-            });
-        }
-
-        private void Investigate(Failure failure, ITimeSet timeSet)
-        {
-            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
-                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
-                $"Checking if node responds to debug/info...");
-
-            try
-            {
-                var debugInfo = GetDebugInfo();
-                if (string.IsNullOrEmpty(debugInfo.Spr))
-                {
-                    Log("Did not get value debug/info response.");
-                    Throw(failure);
-                }
-                else
-                {
-                    Log("Got valid response from debug/info.");
-                }
-            }
-            catch (Exception ex)
-            {
-                Log("Got exception from debug/info call: " + ex);
-                Throw(failure);
-            }
-
-            if (failure.Duration < timeSet.HttpCallTimeout())
-            {
-                Log("Retry failed within HTTP timeout duration.");
-                Throw(failure);
-            }
+            if (processControl.HasCrashed()) throw new Exception($"Container {GetName()} has crashed.");
         }
 
         private void Throw(Failure failure)
@@ -286,7 +294,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{GetName()} {msg}");
+            log.Log($"({GetName()}) {msg}");
         }
     }
 

ProjectPlugins/CodexClient/CodexClient.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApiClient" />
+  </ItemGroup>
+
+<ItemGroup>
+	<PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+	<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	<PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+</ItemGroup>
+	
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\Framework\WebUtils\WebUtils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexInstance.cs

@@ -0,0 +1,54 @@
+using Utils;
+
+namespace CodexClient
+{
+    public interface ICodexInstance
+    {
+        string Name { get; }
+        string ImageName { get; }
+        DateTime StartUtc { get; }
+        Address DiscoveryEndpoint { get; }
+        Address ApiEndpoint { get; }
+        Address ListenEndpoint { get; }
+        EthAccount? EthAccount { get; }
+        Address? MetricsEndpoint { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint)
+        {
+            Name = name;
+            ImageName = imageName;
+            StartUtc = startUtc;
+            DiscoveryEndpoint = discoveryEndpoint;
+            ApiEndpoint = apiEndpoint;
+            ListenEndpoint = listenEndpoint;
+            EthAccount = ethAccount;
+            MetricsEndpoint = metricsEndpoint;
+        }
+
+        public string Name { get; }
+        public string ImageName { get; }
+        public DateTime StartUtc { get; }
+        public Address DiscoveryEndpoint { get; }
+        public Address ApiEndpoint { get; }
+        public Address ListenEndpoint { get; }
+        public EthAccount? EthAccount { get; }
+        public Address? MetricsEndpoint { get; }
+
+        public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint, EthAccount? ethAccount = null)
+        {
+            return new CodexInstance(
+                name,
+                imageName: "-",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: Address.Empty(),
+                apiEndpoint: apiEndpoint,
+                listenEndpoint: Address.Empty(),
+                ethAccount: ethAccount,
+                metricsEndpoint: null
+            );
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexLogLevel.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public enum CodexLogLevel
     {

ProjectPlugins/CodexClient/CodexLogLine.cs

@@ -1,6 +1,6 @@
 using System.Globalization;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class CodexLogLine
     {

ProjectPlugins/CodexClient/CodexNode.cs

@@ -1,98 +1,95 @@
-using CodexPlugin.Hooks;
-using Core;
+using CodexClient.Hooks;
 using FileUtils;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
 using Logging;
-using MetricsPlugin;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
-    public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
+    public partial interface ICodexNode : IHasEthAddress, IHasMetricsScrapeTarget
     {
         string GetName();
+        string GetImageName();
         string GetPeerId();
         DebugInfo GetDebugInfo(bool log = false);
+        void SetLogLevel(string logLevel);
         string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
-        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
-        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
         LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
         DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
-        CrashWatcher CrashWatcher { get; }
-        PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
         EthAccount EthAccount { get; }
+        StoragePurchase? GetPurchaseStatus(string purchaseId);
+
+        Address GetDiscoveryEndpoint();
+        Address GetApiEndpoint();
+        Address GetListenEndpoint();
 
         /// <summary>
         /// Warning! The node is not usable after this.
         /// TODO: Replace with delete-blocks debug call once available in Codex.
         /// </summary>
-        void DeleteRepoFolder();
+        void DeleteDataDirFolder();
         void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(string additionalName = "");
+        bool HasCrashed();
     }
 
     public class CodexNode : ICodexNode
     {
         private const string UploadFailedMessage = "Unable to store block";
         private readonly ILog log;
-        private readonly IPluginTools tools;
         private readonly ICodexNodeHooks hooks;
-        private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
         private string peerId = string.Empty;
         private string nodeId = string.Empty;
+        private readonly CodexAccess codexAccess;
+        private readonly IFileManager fileManager;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
+        public CodexNode(ILog log, CodexAccess codexAccess, IFileManager fileManager, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks)
         {
-            this.tools = tools;
-            this.ethAccount = ethAccount;
-            CodexAccess = codexAccess;
-            Group = group;
+            this.codexAccess = codexAccess;
+            this.fileManager = fileManager;
             Marketplace = marketplaceAccess;
             this.hooks = hooks;
             Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
 
-            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
+            this.log = new LogPrefixer(log, $"{GetName()} ");
         }
 
         public void Awake()
         {
-            hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image, ethAccount);
+            hooks.OnNodeStarting(codexAccess.GetStartUtc(), codexAccess.GetImageName(), codexAccess.GetEthAccount());
         }
 
         public void Initialize()
         {
-            hooks.OnNodeStarted(peerId, nodeId);
+            InitializePeerNodeId();
+            InitializeLogReplacements();
+
+            hooks.OnNodeStarted(this, peerId, nodeId);
         }
 
-        public RunningPod Pod { get { return CodexAccess.Container; } }
-        
-        public RunningContainer Container { get { return Pod.Containers.Single(); } }
-        public CodexAccess CodexAccess { get; }
-        public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
-        public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
         public DebugInfoVersion Version { get; private set; }
         public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
 
-        public IMetricsScrapeTarget MetricsScrapeTarget
+        public StoragePurchase? GetPurchaseStatus(string purchaseId)
         {
-            get
-            {
-                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
-            }
+            return codexAccess.GetPurchaseStatus(purchaseId);
         }
 
         public EthAddress EthAddress 
@@ -100,7 +97,7 @@ namespace CodexPlugin
             get
             {
                 EnsureMarketplace();
-                return ethAccount!.EthAddress;
+                return codexAccess.GetEthAccount()!.EthAddress;
             }
         }
 
@@ -109,13 +106,18 @@ namespace CodexPlugin
             get
             {
                 EnsureMarketplace();
-                return ethAccount!;
+                return codexAccess.GetEthAccount()!;
             }
         }
 
         public string GetName()
         {
-            return Container.Name;
+            return codexAccess.GetName();
+        }
+
+        public string GetImageName()
+        {
+            return codexAccess.GetImageName();
         }
 
         public string GetPeerId()
@@ -125,7 +127,7 @@ namespace CodexPlugin
 
         public DebugInfo GetDebugInfo(bool log = false)
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = codexAccess.GetDebugInfo();
             if (log)
             {
                 var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
@@ -134,27 +136,27 @@ namespace CodexPlugin
             return debugInfo;
         }
 
+        public void SetLogLevel(string logLevel)
+        {
+            codexAccess.SetLogLevel(logLevel);
+        }
+
         public string GetSpr()
         {
-            return CodexAccess.GetSpr();
+            return codexAccess.GetSpr();
         }
 
         public DebugPeer GetDebugPeer(string peerId)
         {
-            return CodexAccess.GetDebugPeer(peerId);
+            return codexAccess.GetDebugPeer(peerId);
         }
 
         public ContentId UploadFile(TrackedFile file)
         {
-            return UploadFile(file, DoNothing);
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"");
         }
 
-        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
-        {
-            return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
-        }
-
-        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -166,7 +168,7 @@ namespace CodexPlugin
             var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(input, onFailure);
+                return codexAccess.UploadFile(input);
             });
 
             var response = measurement.Value;
@@ -184,17 +186,12 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
-        }
-
-        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
-        {
-            var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
+            var file = fileManager.CreateEmptyFile(fileLabel);
             hooks.OnFileDownloading(contentId);
             Log($"Downloading '{contentId}'...");
 
             var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
-            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file));
 
             var size = file.GetFilesize();
             transferSpeeds.AddDownloadSample(size, measurement);
@@ -205,22 +202,39 @@ namespace CodexPlugin
 
         public LocalDataset DownloadStreamless(ContentId cid)
         {
-            return CodexAccess.DownloadStreamless(cid);
+            Log($"Downloading streamless '{cid}' (no-wait)");
+            return codexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = codexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
         }
 
         public LocalDataset DownloadManifestOnly(ContentId cid)
         {
-            return CodexAccess.DownloadManifestOnly(cid);
+            Log($"Downloading manifest-only '{cid}'");
+            return codexAccess.DownloadManifestOnly(cid);
         }
 
         public LocalDatasetList LocalFiles()
         {
-            return CodexAccess.LocalFiles();
+            return codexAccess.LocalFiles();
         }
 
         public CodexSpace Space()
         {
-            return CodexAccess.Space();
+            return codexAccess.Space();
         }
 
         public void ConnectToPeer(ICodexNode node)
@@ -229,47 +243,53 @@ namespace CodexPlugin
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
+            codexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
 
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public PodInfo GetPodInfo()
+        public void DeleteDataDirFolder()
         {
-            return CodexAccess.GetPodInfo();
-        }
-
-        public void DeleteRepoFolder()
-        {
-            CodexAccess.DeleteRepoFolder();
+            codexAccess.DeleteDataDirFolder();
         }
 
         public void Stop(bool waitTillStopped)
         {
             Log("Stopping...");
             hooks.OnNodeStopping();
-
-            CrashWatcher.Stop();
-            Group.Stop(this, waitTillStopped);
+            codexAccess.Stop(waitTillStopped);
         }
 
-        public void EnsureOnlineGetVersionResponse()
+        public IDownloadedLog DownloadLog(string additionalName = "")
         {
-            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            peerId = debugInfo.Id;
-            nodeId = debugInfo.Table.LocalNode.NodeId;
-            var nodeName = CodexAccess.Container.Name;
+            return codexAccess.DownloadLog(additionalName);
+        }
 
-            if (!debugInfo.Version.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
-            }
+        public Address GetDiscoveryEndpoint()
+        {
+            return codexAccess.GetDiscoveryEndpoint();
+        }
 
-            log.AddStringReplace(peerId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
-            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(debugInfo.Table.LocalNode.NodeId), nodeName);
-            Version = debugInfo.Version;
+        public Address GetApiEndpoint()
+        {
+            return codexAccess.GetApiEndpoint();
+        }
+
+        public Address GetListenEndpoint()
+        {
+            return codexAccess.GetListenEndpoint();
+        }
+
+        public Address GetMetricsScrapeTarget()
+        {
+            var address = codexAccess.GetMetricsEndpoint();
+            if (address == null) throw new Exception("Metrics ScrapeTarget accessed, but node was not started with EnableMetrics()");
+            return address;
+        }
+
+        public bool HasCrashed()
+        {
+            return codexAccess.HasCrashed();
         }
 
         public override string ToString()
@@ -277,22 +297,44 @@ namespace CodexPlugin
             return $"CodexNode:{GetName()}";
         }
 
+        private void InitializePeerNodeId()
+        {
+            var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online");
+            if (!debugInfo.Version.IsValid())
+            {
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
+            }
+
+            peerId = debugInfo.Id;
+            nodeId = debugInfo.Table.LocalNode.NodeId;
+            Version = debugInfo.Version;
+        }
+
+        private void InitializeLogReplacements()
+        {
+            var nodeName = GetName();
+
+            log.AddStringReplace(peerId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
+            log.AddStringReplace(nodeId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName);
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Pod);
+            var peerId = peer.GetDiscoveryEndpoint().Host
+                .Replace("http://", "")
+                .Replace("https://", "");
 
             return peerInfo.Addrs.Select(a => a
-                .Replace("0.0.0.0", podInfo.Ip))
+                .Replace("0.0.0.0", peerId))
                 .ToArray();
         }
 
-        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
+        private void DownloadToFile(string contentId, TrackedFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);
-            var timeout = tools.TimeSet.HttpCallTimeout();
+            var timeout = TimeSpan.FromMinutes(2.0); // todo: make this user-controllable.
             try
             {
                 // Type of stream generated by openAPI client does not support timeouts.
@@ -300,7 +342,7 @@ namespace CodexPlugin
                 var cts = new CancellationTokenSource();
                 var downloadTask = Task.Run(() =>
                 {
-                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    using var downloadStream = codexAccess.DownloadFile(contentId);
                     downloadStream.CopyTo(fileStream);
                 }, cts.Token);
                 
@@ -314,25 +356,54 @@ namespace CodexPlugin
                 cts.Cancel();
                 throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
-            catch
+            catch (Exception ex)
             {
-                Log($"Failed to download file '{contentId}'.");
+                Log($"Failed to download file '{contentId}': {ex}");
                 throw;
             }
         }
 
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
+        }
+
         private void EnsureMarketplace()
         {
-            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
+            if (codexAccess.GetEthAccount() == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
         }
 
         private void Log(string msg)
         {
             log.Log(msg);
         }
-
-        private void DoNothing(Failure failure)
-        {
-        }
     }
 }

ProjectPlugins/CodexClient/CodexNodeFactory.cs

@@ -0,0 +1,52 @@
+using CodexClient.Hooks;
+using FileUtils;
+using Logging;
+using WebUtils;
+
+namespace CodexClient
+{
+    public class CodexNodeFactory
+    {
+        private readonly ILog log;
+        private readonly IFileManager fileManager;
+        private readonly CodexHooksFactory hooksFactory;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControlFactory processControlFactory;
+
+        public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory)
+        {
+            this.log = log;
+            this.fileManager = fileManager;
+            this.hooksFactory = hooksFactory;
+            this.httpFactory = httpFactory;
+            this.processControlFactory = processControlFactory;
+        }
+
+        public CodexNodeFactory(ILog log, HttpFactory httpFactory, string dataDir)
+            : this(log, new FileManager(log, dataDir), new CodexHooksFactory(), httpFactory, new DoNothingProcessControlFactory())
+        {
+        }
+
+        public CodexNodeFactory(ILog log, string dataDir)
+            : this(log, new HttpFactory(log), dataDir)
+        {
+        }
+
+        public ICodexNode CreateCodexNode(ICodexInstance instance)
+        {
+            var processControl = processControlFactory.CreateProcessControl(instance);
+            var access = new CodexAccess(log, httpFactory, processControl, instance);
+            var hooks = hooksFactory.CreateHooks(access.GetName());
+            var marketplaceAccess = CreateMarketplaceAccess(instance, access, hooks);
+            var node =  new CodexNode(log, access, fileManager, marketplaceAccess, hooks);
+            node.Initialize();
+            return node;
+        }
+
+        private IMarketplaceAccess CreateMarketplaceAccess(ICodexInstance instance, CodexAccess access, ICodexNodeHooks hooks)
+        {
+            if (instance.EthAccount == null) return new MarketplaceUnavailable();
+            return new MarketplaceAccess(log, access, hooks);
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexTypes.cs

@@ -1,7 +1,7 @@
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class DebugInfo
     {
@@ -109,6 +109,16 @@ namespace CodexPlugin
         {
             return HashCode.Combine(Id);
         }
+
+        public static bool operator ==(ContentId a, ContentId b)
+        {
+            return a.Id == b.Id;
+        }
+
+        public static bool operator !=(ContentId a, ContentId b)
+        {
+            return a.Id != b.Id;
+        }
     }
 
     public class CodexSpace

ProjectPlugins/CodexClient/CodexUtils.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public static class CodexUtils
     {

ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs

@@ -1,7 +1,6 @@
-using GethPlugin;
-using Utils;
+using Utils;
 
-namespace CodexPlugin.Hooks
+namespace CodexClient.Hooks
 {
     public interface ICodexHooksProvider
     {
@@ -10,11 +9,14 @@ namespace CodexPlugin.Hooks
 
     public class CodexHooksFactory
     {
-        public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
+        public List<ICodexHooksProvider> Providers { get; } = new List<ICodexHooksProvider>();
 
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
-            return Provider.CreateHooks(nodeName);
+            if (Providers.Count == 0) return new DoNothingCodexHooks();
+
+            var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray();
+            return new MuxingCodexNodeHooks(hooks);
         }
     }
 
@@ -44,7 +46,7 @@ namespace CodexPlugin.Hooks
         {
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
         }
 

ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs

@@ -0,0 +1,78 @@
+using Utils;
+
+namespace CodexClient.Hooks
+{
+    public interface ICodexNodeHooks
+    {
+        void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
+        void OnNodeStarted(ICodexNode node, string peerId, string nodeId);
+        void OnNodeStopping();
+        void OnFileUploading(string uid, ByteSize size);
+        void OnFileUploaded(string uid, ByteSize size, ContentId cid);
+        void OnFileDownloading(ContentId cid);
+        void OnFileDownloaded(ByteSize size, ContentId cid);
+        void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
+        void OnStorageContractUpdated(StoragePurchase purchaseStatus);
+        void OnStorageAvailabilityCreated(StorageAvailability response);
+    }
+
+    public class MuxingCodexNodeHooks : ICodexNodeHooks
+    {
+        private readonly ICodexNodeHooks[] backingHooks;
+
+        public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks)
+        {
+            this.backingHooks = backingHooks;
+        }
+
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloaded(size, cid);
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloading(cid);
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid);
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+            foreach (var h in backingHooks) h.OnFileUploading(uid, size);
+        }
+
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId);
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount);
+        }
+
+        public void OnNodeStopping()
+        {
+            foreach (var h in backingHooks) h.OnNodeStopping();
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+            foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response);
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract);
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus);
+        }
+    }
+}

ProjectPlugins/CodexClient/Mapper.cs

@@ -1,10 +1,9 @@
-using CodexContractsPlugin;
-using CodexOpenApi;
+using CodexOpenApi;
 using Newtonsoft.Json.Linq;
 using System.Numerics;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class Mapper
     {
@@ -15,7 +14,7 @@ namespace CodexPlugin
                 Id = debugInfo.Id,
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
-                AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
+                AnnounceAddresses = debugInfo.AnnounceAddresses.ToArray(),
                 Version = Map(debugInfo.Codex),
                 Table = Map(debugInfo.Table)
             };
@@ -43,8 +42,8 @@ namespace CodexPlugin
             return new CodexOpenApi.SalesAvailabilityCREATE
             {
                 Duration = ToDecInt(availability.MaxDuration.TotalSeconds),
-                MinPrice = ToDecInt(availability.MinPriceForTotalSpace),
-                MaxCollateral = ToDecInt(availability.MaxCollateral),
+                MinPricePerBytePerSecond = ToDecInt(availability.MinPricePerBytePerSecond),
+                TotalCollateral = ToDecInt(availability.TotalCollateral),
                 TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes)
             };
         }
@@ -55,27 +54,27 @@ namespace CodexPlugin
             {
                 Duration = ToDecInt(purchase.Duration.TotalSeconds),
                 ProofProbability = ToDecInt(purchase.ProofProbability),
-                Reward = ToDecInt(purchase.PricePerSlotPerSecond),
-                Collateral = ToDecInt(purchase.RequiredCollateral),
+                PricePerBytePerSecond = ToDecInt(purchase.PricePerBytePerSecond),
+                CollateralPerByte = ToDecInt(purchase.CollateralPerByte),
                 Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
                 Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
                 Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };
         }
 
-        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        public StorageAvailability[] Map(ICollection<CodexOpenApi.SalesAvailabilityREAD> availabilities)
         {
             return availabilities.Select(a => Map(a)).ToArray();
         }
 
-        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD availability)
         {
             return new StorageAvailability
             (
                 ToByteSize(availability.TotalSize),
                 ToTimespan(availability.Duration),
-                new TestToken(ToBigIng(availability.MinPrice)),
-                new TestToken(ToBigIng(availability.MaxCollateral))
+                new TestToken(ToBigIng(availability.MinPricePerBytePerSecond)),
+                new TestToken(ToBigIng(availability.TotalCollateral))
             )
             {
                 Id = availability.Id,
@@ -83,49 +82,80 @@ namespace CodexPlugin
             };
         }
 
-        // TODO: Fix openapi spec for this call.
-        //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
-        //{
-        //    return new StoragePurchase(Map(purchase.Request))
-        //    {
-        //        State = purchase.State,
-        //        Error = purchase.Error
-        //    };
-        //}
+        public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        {
+            return new StoragePurchase
+            {
+                Request = Map(purchase.Request),
+                State = purchase.State.ToString(), //Map(purchase.State),
+                Error = purchase.Error
+            };
+        }
 
-        //public StorageRequest Map(CodexOpenApi.StorageRequest request)
-        //{
-        //    return new StorageRequest(Map(request.Ask), Map(request.Content))
-        //    {
-        //        Id = request.Id,
-        //        Client = request.Client,
-        //        Expiry = TimeSpan.FromSeconds(Convert.ToInt64(request.Expiry)),
-        //        Nonce = request.Nonce
-        //    };
-        //}
+        public StoragePurchaseState Map(PurchaseState purchaseState)
+        {
+            // TODO: to be re-enabled when marketplace api lines up with openapi.yaml.
 
-        //public StorageAsk Map(CodexOpenApi.StorageAsk ask)
-        //{
-        //    return new StorageAsk
-        //    {
-        //        Duration = TimeSpan.FromSeconds(Convert.ToInt64(ask.Duration)),
-        //        MaxSlotLoss = ask.MaxSlotLoss,
-        //        ProofProbability = ask.ProofProbability,
-        //        Reward = Convert.ToDecimal(ask.Reward).TstWei(),
-        //        Slots = ask.Slots,
-        //        SlotSize = new ByteSize(Convert.ToInt64(ask.SlotSize))
-        //    };
-        //}
+            // Explicit mapping: If the API changes, we will get compile errors here.
+            // That's what we want.
+            switch (purchaseState)
+            {
+                case PurchaseState.Cancelled:
+                    return StoragePurchaseState.Cancelled;
+                case PurchaseState.Error:
+                    return StoragePurchaseState.Error;
+                case PurchaseState.Failed:
+                    return StoragePurchaseState.Failed;
+                case PurchaseState.Finished:
+                    return StoragePurchaseState.Finished;
+                case PurchaseState.Pending:
+                    return StoragePurchaseState.Pending;
+                case PurchaseState.Started:
+                    return StoragePurchaseState.Started;
+                case PurchaseState.Submitted:
+                    return StoragePurchaseState.Submitted;
+                case PurchaseState.Unknown:
+                    return StoragePurchaseState.Unknown;
+            }
 
-        //public StorageContent Map(CodexOpenApi.Content content)
-        //{
-        //    return new StorageContent
-        //    {
-        //        Cid = content.Cid
-        //    };
-        //}
+            throw new Exception("API incompatibility detected. Unknown purchaseState: " + purchaseState.ToString());
+        }
 
-        public CodexSpace Map(Space space)
+        public StorageRequest Map(CodexOpenApi.StorageRequest request)
+        {
+            return new StorageRequest
+            {
+                Ask = Map(request.Ask),
+                Content = Map(request.Content),
+                Id = request.Id,
+                Client = request.Client,
+                Expiry = request.Expiry,
+                Nonce = request.Nonce
+            };
+        }
+
+        public StorageAsk Map(CodexOpenApi.StorageAsk ask)
+        {
+            return new StorageAsk
+            {
+                Duration = ask.Duration,
+                MaxSlotLoss = ask.MaxSlotLoss,
+                ProofProbability = ask.ProofProbability,
+                PricePerBytePerSecond = ask.PricePerBytePerSecond,
+                Slots = ask.Slots,
+                SlotSize = ask.SlotSize
+            };
+        }
+
+        public StorageContent Map(CodexOpenApi.Content content)
+        {
+            return new StorageContent
+            {
+                Cid = content.Cid
+            };
+        }
+
+        public CodexSpace Map(CodexOpenApi.Space space)
         {
             return new CodexSpace
             {
@@ -136,7 +166,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion Map(CodexVersion obj)
+        private DebugInfoVersion Map(CodexOpenApi.CodexVersion obj)
         {
             return new DebugInfoVersion
             {
@@ -145,7 +175,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTable Map(PeersTable obj)
+        private DebugInfoTable Map(CodexOpenApi.PeersTable obj)
         {
             return new DebugInfoTable
             {
@@ -154,7 +184,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode Map(Node? token)
+        private DebugInfoTableNode Map(CodexOpenApi.Node? token)
         {
             if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
@@ -167,7 +197,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
+        private DebugInfoTableNode[] Map(ICollection<CodexOpenApi.Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {

ProjectPlugins/CodexClient/MarketplaceAccess.cs

@@ -1,8 +1,8 @@
-using CodexPlugin.Hooks;
+using CodexClient.Hooks;
 using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IMarketplaceAccess
     {
@@ -27,8 +27,12 @@ namespace CodexPlugin
         public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
+            var swResult = Stopwatch.Measure(log, nameof(RequestStorage), () =>
+            {
+                return codexAccess.RequestStorage(purchase);
+            });
 
-            var response = codexAccess.RequestStorage(purchase);
+            var response = swResult.Value;
 
             if (string.IsNullOrEmpty(response) ||
                 response == "Unable to encode manifest" ||
@@ -42,12 +46,7 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
-            contract.WaitForStorageContractSubmitted();
-
-            hooks.OnStorageContractSubmitted(contract);
-
-            return contract;
+            return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
@@ -72,7 +71,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
+            log.Log($"{codexAccess.GetName()} {msg}");
         }
     }
 

ProjectPlugins/CodexClient/MarketplaceTypes.cs

@@ -1,8 +1,7 @@
-using CodexContractsPlugin;
-using Logging;
+using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class StoragePurchaseRequest
     {
@@ -12,8 +11,8 @@ namespace CodexPlugin
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
-        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
+        public TestToken PricePerBytePerSecond { get; set; } = 1.TstWei();
+        public TestToken CollateralPerByte { get; set; } = 1.TstWei();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }
@@ -23,8 +22,8 @@ namespace CodexPlugin
         public void Log(ILog log)
         {
             log.Log($"Requesting storage for: {ContentId.Id}... (" +
-                $"pricePerSlotPerSecond: {PricePerSlotPerSecond}, " +
-                $"requiredCollateral: {RequiredCollateral}, " +
+                $"pricePerBytePerSecond: {PricePerBytePerSecond}, " +
+                $"collateralPerByte: {CollateralPerByte}, " +
                 $"minRequiredNumberOfNodes: {MinRequiredNumberOfNodes}, " +
                 $"nodeFailureTolerance: {NodeFailureTolerance}, " +
                 $"proofProbability: {ProofProbability}, " +
@@ -38,6 +37,24 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
+    }
+
+    public enum StoragePurchaseState
+    {
+        Cancelled = 0,
+        Error = 1,
+        Failed = 2,
+        Finished = 3,
+        Pending = 4,
+        Started = 5,
+        Submitted = 6,
+        Unknown = 7,
     }
 
     public class StorageRequest
@@ -56,7 +73,7 @@ namespace CodexPlugin
         public string SlotSize { get; set; } = string.Empty;
         public string Duration { get; set; } = string.Empty;
         public string ProofProbability { get; set; } = string.Empty;
-        public string Reward { get; set; } = string.Empty;
+        public string PricePerBytePerSecond { get; set; } = string.Empty;
         public int MaxSlotLoss { get; set; }
     }
 
@@ -69,19 +86,19 @@ namespace CodexPlugin
 
     public class StorageAvailability
     {
-        public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral)
+        public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
         {
             TotalSpace = totalSpace;
             MaxDuration = maxDuration;
-            MinPriceForTotalSpace = minPriceForTotalSpace;
-            MaxCollateral = maxCollateral;
+            MinPricePerBytePerSecond = minPricePerBytePerSecond;
+            TotalCollateral = totalCollateral;
         }
 
         public string Id { get; set; } = string.Empty;
         public ByteSize TotalSpace { get; }
         public TimeSpan MaxDuration { get; }
-        public TestToken MinPriceForTotalSpace { get; }
-        public TestToken MaxCollateral { get; } 
+        public TestToken MinPricePerBytePerSecond { get; }
+        public TestToken TotalCollateral { get; } 
         public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
@@ -89,8 +106,8 @@ namespace CodexPlugin
             log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
-                $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +
-                $"maxCollateral: {MaxCollateral})");
+                $"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
+                $"totalCollateral: {TotalCollateral})");
         }
     }
 }

ProjectPlugins/CodexClient/ProcessControl.cs

@@ -0,0 +1,46 @@
+using Logging;
+
+namespace CodexClient
+{
+    public interface IProcessControlFactory
+    {
+        IProcessControl CreateProcessControl(ICodexInstance instance);
+    }
+
+    public interface IProcessControl
+    {
+        void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(LogFile file);
+        void DeleteDataDirFolder();
+        bool HasCrashed();
+    }
+
+    public class DoNothingProcessControlFactory : IProcessControlFactory
+    {
+        public IProcessControl CreateProcessControl(ICodexInstance instance)
+        {
+            return new DoNothingProcessControl();
+        }
+    }
+
+    public class DoNothingProcessControl : IProcessControl
+    {
+        public void DeleteDataDirFolder()
+        {
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            throw new NotImplementedException("Not supported by DoNothingProcessControl");
+        }
+
+        public bool HasCrashed()
+        {
+            return false;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+        }
+    }
+}

ProjectPlugins/CodexClient/StoragePurchaseContract.cs

@@ -1,20 +1,20 @@
-using CodexContractsPlugin;
-using CodexPlugin.Hooks;
-using GethPlugin;
+using CodexClient.Hooks;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IStoragePurchaseContract
     {
         string PurchaseId { get; }
         StoragePurchaseRequest Purchase { get; }
         ContentId ContentId { get; }
+        StoragePurchase? GetStatus();
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished(ICodexContracts contracts);
+        void WaitForStorageContractFinished();
+        void WaitForContractFailed();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -22,11 +22,13 @@ namespace CodexPlugin
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
         private readonly ICodexNodeHooks hooks;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60);
         private readonly DateTime contractPendingUtc = DateTime.UtcNow;
         private DateTime? contractSubmittedUtc = DateTime.UtcNow;
         private DateTime? contractStartedUtc;
         private DateTime? contractFinishedUtc;
+        private string lastState = string.Empty;
+        private ContentId encodedContentId = new ContentId();
 
         public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase, ICodexNodeHooks hooks)
         {
@@ -35,21 +37,40 @@ namespace CodexPlugin
             PurchaseId = purchaseId;
             Purchase = purchase;
             this.hooks = hooks;
-            ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
         }
 
         public string PurchaseId { get; }
         public StoragePurchaseRequest Purchase { get; }
-        public ContentId ContentId { get; }
+        public ContentId ContentId
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(encodedContentId.Id)) GetStatus();
+                return encodedContentId;
+            }
+        }
 
         public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
         public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
         public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
 
+        public StoragePurchase? GetStatus()
+        {
+            var status = codexAccess.GetPurchaseStatus(PurchaseId);
+            if (status != null)
+            {
+                encodedContentId = new ContentId(status.Request.Content.Cid);
+            }
+            return status;
+        }
+
         public void WaitForStorageContractSubmitted()
         {
-            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
+            var timeout = Purchase.Expiry + gracePeriod;
+            var raiseHook = lastState != "submitted";
+            WaitForStorageContractState(timeout, "submitted", sleep: 200);
             contractSubmittedUtc = DateTime.UtcNow;
+            if (raiseHook) hooks.OnStorageContractSubmitted(this);
             LogSubmittedDuration();
             AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
         }
@@ -64,7 +85,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished(ICodexContracts contracts)
+        public void WaitForStorageContractFinished()
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -76,23 +97,21 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
-
-            contracts.WaitUntilNextPeriod();
-            contracts.WaitUntilNextPeriod();
-
-            var blocks = 3;
-            Log($"Waiting {blocks} blocks for nodes to process payouts...");
-            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
         }
 
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        public void WaitForContractFailed()
         {
-            return codexAccess.GetPurchaseStatus(purchaseId);
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "failed");
         }
 
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
         {
-            var lastState = "";
             var waitStart = DateTime.UtcNow;
 
             Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
@@ -124,7 +143,7 @@ namespace CodexPlugin
         private void LogSubmittedDuration()
         {
             Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
-                $"( < {Time.FormatDuration(gracePeriod)})");
+                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
         }
 
         private void LogStartedDuration()

ProjectPlugins/CodexClient/TransferSpeeds.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface ITransferSpeeds
     {

ProjectPlugins/CodexClient/openapi.yaml

@@ -6,7 +6,7 @@ info:
   description: "List of endpoints and interfaces available to Codex API users"
 
 security:
-  - { }
+  - {}
 
 components:
   schemas:
@@ -50,9 +50,9 @@ components:
       type: string
       description: Address of Ethereum address
 
-    Reward:
+    PricePerBytePerSecond:
       type: string
-      description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+      description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay
 
     Duration:
       type: string
@@ -138,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        announceAddresses:
+          type: array
+          items:
+            $ref: "#/components/schemas/MultiAddress"
         table:
           $ref: "#/components/schemas/PeersTable"
         codex:
@@ -153,12 +157,12 @@ components:
           description: Total size of availability's storage in bytes as decimal string
         duration:
           $ref: "#/components/schemas/Duration"
-        minPrice:
+        minPricePerBytePerSecond:
           type: string
-          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
-        maxCollateral:
+          description: Minimal price per byte per second paid (in amount of tokens) for the hosted request's slot for the request's duration as decimal string
+        totalCollateral:
           type: string
-          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
+          description: Total collateral (in amount of tokens) that can be used for matching requests
 
     SalesAvailabilityREAD:
       allOf:
@@ -174,8 +178,8 @@ components:
         - $ref: "#/components/schemas/SalesAvailability"
         - required:
             - totalSize
-            - minPrice
-            - maxCollateral
+            - minPricePerBytePerSecond
+            - totalCollateral
             - duration
 
     Slot:
@@ -239,16 +243,16 @@ components:
     StorageRequestCreation:
       type: object
       required:
-        - reward
+        - pricePerBytePerSecond
         - duration
         - proofProbability
-        - collateral
+        - collateralPerByte
         - expiry
       properties:
         duration:
           $ref: "#/components/schemas/Duration"
-        reward:
-          $ref: "#/components/schemas/Reward"
+        pricePerBytePerSecond:
+          $ref: "#/components/schemas/PricePerBytePerSecond"
         proofProbability:
           $ref: "#/components/schemas/ProofProbability"
         nodes:
@@ -259,16 +263,16 @@ components:
           description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
           type: integer
           default: 0
-        collateral:
+        collateralPerByte:
           type: string
-          description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
+          description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots
         expiry:
           type: string
           description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
     StorageAsk:
       type: object
       required:
-        - reward
+        - pricePerBytePerSecond
       properties:
         slots:
           description: Number of slots (eq. hosts) that the Request want to have the content spread over
@@ -280,8 +284,8 @@ components:
           $ref: "#/components/schemas/Duration"
         proofProbability:
           $ref: "#/components/schemas/ProofProbability"
-        reward:
-          $ref: "#/components/schemas/Reward"
+        pricePerBytePerSecond:
+          $ref: "#/components/schemas/PricePerBytePerSecond"
         maxSlotLoss:
           type: integer
           description: Max slots that can be lost without data considered to be lost
@@ -321,6 +325,7 @@ components:
             - unknown
         error:
           type: string
+          nullable: true
           description: If Request failed, then here is presented the error message
         request:
           $ref: "#/components/schemas/StorageRequest"
@@ -367,12 +372,6 @@ components:
           nullable: true
           description: "The original mimetype of the uploaded content (optional)"
           example: image/png
-        uploadedAt:
-          type: integer
-          format: int64
-          nullable: true
-          description: "The UTC upload timestamp in seconds"
-          example: 1729244192
 
     Space:
       type: object
@@ -414,14 +413,14 @@ paths:
       description: |
         If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
         to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
-      tags: [ Node ]
+      tags: [Node]
       operationId: connectPeer
       parameters:
         - in: path
           name: peerId
           required: true
           schema:
-              $ref: "#/components/schemas/PeerId"
+            $ref: "#/components/schemas/PeerId"
           description: Peer that should be dialed.
         - in: query
           name: addrs
@@ -444,7 +443,7 @@ paths:
   "/data":
     get:
       summary: "Lists manifest CIDs stored locally in node."
-      tags: [ Data ]
+      tags: [Data]
       operationId: listData
       responses:
         "200":
@@ -464,7 +463,7 @@ paths:
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
-      tags: [ Data ]
+      tags: [Data]
       operationId: upload
       parameters:
         - name: content-type
@@ -480,7 +479,7 @@ paths:
           description: The content disposition used to send the filename.
           schema:
             type: string
-            example: "attachment; filename=\"codex.png\""
+            example: 'attachment; filename="codex.png"'
       requestBody:
         content:
           application/octet-stream:
@@ -500,14 +499,14 @@ paths:
   "/data/{cid}":
     get:
       summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadLocal
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: File to be downloaded.
 
       responses:
@@ -528,14 +527,14 @@ paths:
   "/data/{cid}/network":
     post:
       summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetwork
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File to be downloaded."
       responses:
         "200":
@@ -554,14 +553,14 @@ paths:
   "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File to be downloaded."
       responses:
         "200":
@@ -581,14 +580,14 @@ paths:
   "/data/{cid}/network/manifest":
     get:
       summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetworkManifest
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File for which the manifest is to be downloaded."
       responses:
         "200":
@@ -607,7 +606,7 @@ paths:
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
-      tags: [ Data ]
+      tags: [Data]
       operationId: space
       responses:
         "200":
@@ -623,7 +622,7 @@ paths:
   "/sales/slots":
     get:
       summary: "Returns active slots"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getActiveSlots
       responses:
         "200":
@@ -641,7 +640,7 @@ paths:
   "/sales/slots/{slotId}":
     get:
       summary: "Returns active slot with id {slotId} for the host"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getActiveSlotById
       parameters:
         - in: path
@@ -670,7 +669,7 @@ paths:
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getAvailabilities
       responses:
         "200":
@@ -689,7 +688,7 @@ paths:
     post:
       summary: "Offers storage for sale"
       operationId: offerStorage
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       requestBody:
         content:
           application/json:
@@ -717,7 +716,7 @@ paths:
         The new parameters will be only considered for new requests.
         Existing Requests linked to this Availability will continue as is.
       operationId: updateOfferedStorage
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       parameters:
         - in: path
           name: id
@@ -749,7 +748,7 @@ paths:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       parameters:
         - in: path
           name: id
@@ -778,7 +777,7 @@ paths:
   "/storage/request/{cid}":
     post:
       summary: "Creates a new Request for storage"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: createStorageRequest
       parameters:
         - in: path
@@ -809,7 +808,7 @@ paths:
   "/storage/purchases":
     get:
       summary: "Returns list of purchase IDs"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getPurchases
       responses:
         "200":
@@ -826,7 +825,7 @@ paths:
   "/storage/purchases/{id}":
     get:
       summary: "Returns purchase details"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getPurchase
       parameters:
         - in: path
@@ -853,7 +852,7 @@ paths:
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
-      tags: [ Node ]
+      tags: [Node]
       responses:
         "200":
           description: Node's SPR
@@ -871,7 +870,7 @@ paths:
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
-      tags: [ Node ]
+      tags: [Node]
       responses:
         "200":
           description: Node's Peer ID
@@ -886,7 +885,7 @@ paths:
   "/debug/chronicles/loglevel":
     post:
       summary: "Set log level at run time"
-      tags: [ Debug ]
+      tags: [Debug]
       operationId: setDebugLogLevel
 
       parameters:
@@ -908,7 +907,7 @@ paths:
     get:
       summary: "Gets node information"
       operationId: getDebugInfo
-      tags: [ Debug ]
+      tags: [Debug]
       responses:
         "200":
           description: Node's information

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -14,7 +14,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
             SlotFreedEventDTO[] slotFreed,
-            SlotReservationsFullEventDTO[] slotReservationsFull
+            SlotReservationsFullEventDTO[] slotReservationsFull,
+            ProofSubmittedEventDTO[] proofSubmitted
             )
         {
             BlockInterval = blockInterval;
@@ -25,8 +26,8 @@ namespace CodexContractsPlugin.ChainMonitor
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
             SlotReservationsFull = slotReservationsFull;
-
-            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
+            ProofSubmitted = proofSubmitted;
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull, ProofSubmitted);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -37,6 +38,7 @@ namespace CodexContractsPlugin.ChainMonitor
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
         public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public ProofSubmittedEventDTO[] ProofSubmitted { get; }
         public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
@@ -59,7 +61,8 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
                 events.GetSlotFreedEvents(),
-                events.GetSlotReservationsFull()
+                events.GetSlotReservationsFullEvents(),
+                events.GetProofSubmittedEvents()
             );
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,7 +1,7 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 
@@ -17,7 +17,7 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
         void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
-
+        void OnProofSubmitted(BlockTimeEntry block, string id);
         void OnError(string msg);
     }
 
@@ -39,17 +39,21 @@ namespace CodexContractsPlugin.ChainMonitor
         private readonly ILog log;
         private readonly ICodexContracts contracts;
         private readonly IChainStateChangeHandler handler;
+        private readonly bool doProofPeriodMonitoring;
 
-        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
+        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring)
         {
             this.log = new LogPrefixer(log, "(ChainState) ");
             this.contracts = contracts;
             handler = changeHandler;
+            this.doProofPeriodMonitoring = doProofPeriodMonitoring;
             TotalSpan = new TimeRange(startUtc, startUtc);
+            PeriodMonitor = new PeriodMonitor(this.log, contracts);
         }
 
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
+        public PeriodMonitor PeriodMonitor { get; }
 
         public int Update()
         {
@@ -88,11 +92,18 @@ namespace CodexContractsPlugin.ChainMonitor
             {
                 var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
                 ApplyEvents(b, blockEvents, eventUtc);
+                UpdatePeriodMonitor(b, eventUtc);
 
                 eventUtc += spanPerBlock;
             }
         }
 
+        private void UpdatePeriodMonitor(ulong blockNumber, DateTime eventUtc)
+        {
+            if (!doProofPeriodMonitoring) return;
+            PeriodMonitor.Update(blockNumber, eventUtc, Requests);
+        }
+
         private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
         {
             foreach (var e in blockEvents)
@@ -165,6 +176,13 @@ namespace CodexContractsPlugin.ChainMonitor
             handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(ProofSubmittedEventDTO @event)
+        {
+            var id = Base58.Encode(@event.Id);
+            log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id})");
+            handler.OnProofSubmitted(@event.Block, id);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,5 +1,6 @@
-using GethPlugin;
+using BlockchainUtils;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -56,5 +57,10 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnError(msg);
         }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+            foreach (var handler in Handlers) handler.OnProofSubmitted(block, id);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -1,6 +1,6 @@
 using CodexContractsPlugin.Marketplace;
-using GethPlugin;
 using Logging;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -1,5 +1,6 @@
-using GethPlugin;
+using BlockchainUtils;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -40,5 +41,9 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnError(string msg)
         {
         }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs

@@ -0,0 +1,99 @@
+using Logging;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class PeriodMonitor
+    {
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly List<PeriodReport> reports = new List<PeriodReport>();
+        private ulong? currentPeriod = null;
+
+        public PeriodMonitor(ILog log, ICodexContracts contracts)
+        {
+            this.log = log;
+            this.contracts = contracts;
+        }
+
+        public void Update(ulong blockNumber, DateTime eventUtc, IChainStateRequest[] requests)
+        {
+            var period = contracts.GetPeriodNumber(eventUtc);
+            if (!currentPeriod.HasValue)
+            {
+                currentPeriod = period;
+                return;
+            }
+            if (period == currentPeriod.Value) return;
+
+            CreateReportForPeriod(blockNumber - 1, currentPeriod.Value, requests);
+            currentPeriod = period;
+        }
+
+        public PeriodReport[] GetAndClearReports()
+        {
+            var result = reports.ToArray();
+            reports.Clear();
+            return result;
+        }
+
+        private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests)
+        {
+            log.Log("Creating report for period " + periodNumber);
+
+            ulong total = 0;
+            ulong required = 0;
+            var missed = new List<PeriodProofMissed>();
+            foreach (var request in requests)
+            {
+                for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
+                {
+                    var state = contracts.GetProofState(request.Request, slotIndex, lastBlockInPeriod, periodNumber);
+
+                    total++;
+                    if (state.Required)
+                    {
+                        required++;
+                        if (state.Missing)
+                        {
+                            var idx = Convert.ToInt32(slotIndex);
+                            var host = request.Hosts.GetHost(idx);
+                            missed.Add(new PeriodProofMissed(host, request, idx));
+                        }
+                    }
+                }
+            }
+            reports.Add(new PeriodReport(periodNumber, total, required, missed.ToArray()));
+        }
+    }
+
+    public class PeriodReport
+    {
+        public PeriodReport(ulong periodNumber, ulong totalNumSlots, ulong totalProofsRequired, PeriodProofMissed[] missedProofs)
+        {
+            PeriodNumber = periodNumber;
+            TotalNumSlots = totalNumSlots;
+            TotalProofsRequired = totalProofsRequired;
+            MissedProofs = missedProofs;
+        }
+
+        public ulong PeriodNumber { get; }
+        public ulong TotalNumSlots { get; }
+        public ulong TotalProofsRequired { get; }
+        public PeriodProofMissed[] MissedProofs { get; }
+    }
+
+    public class PeriodProofMissed
+    {
+        public PeriodProofMissed(EthAddress? host, IChainStateRequest request, int slotIndex)
+        {
+            Host = host;
+            Request = request;
+            SlotIndex = slotIndex;
+        }
+
+        public EthAddress? Host { get; }
+        public IChainStateRequest Request { get; }
+        public int SlotIndex { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,7 +1,10 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
@@ -25,7 +28,21 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        ulong GetPeriodNumber(DateTime utc);
         void WaitUntilNextPeriod();
+        ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
+    }
+
+    public class ProofState
+    {
+        public ProofState(bool required, bool missing)
+        {
+            Required = required;
+            Missing = missing;
+        }
+
+        public bool Required { get; }
+        public bool Missing { get; }
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -88,19 +105,23 @@ namespace CodexContractsPlugin
             return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
         }
 
-        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        public byte[] GetSlotId(Request request, decimal slotIndex)
         {
             var encoder = new ABIEncode();
             var encoded = encoder.GetABIEncoded(
-                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("bytes32", request.RequestId),
                 new ABIValue("uint256", slotIndex.ToBig())
             );
 
-            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+            return Sha3Keccack.Current.CalculateHash(encoded);
+        }
 
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var slotId = GetSlotId(storageRequest, slotIndex);
             var func = new GetHostFunction
             {
-                SlotId = hashed
+                SlotId = slotId
             };
             var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
             if (string.IsNullOrEmpty(address)) return null;
@@ -116,6 +137,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public ulong GetPeriodNumber(DateTime utc)
+        {
+            DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);
+            var now = utco.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var result = now / periodSeconds;
+            return Convert.ToUInt64(result);
+        }
+
         public void WaitUntilNextPeriod()
         {
             log.Log("Waiting until next proof period...");
@@ -125,6 +155,52 @@ namespace CodexContractsPlugin
             Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
         }
 
+        public ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period)
+        {
+            var slotId = GetSlotId(storageRequest, slotIndex);
+
+            var required = IsProofRequired(slotId, blockNumber);
+            if (!required) return new ProofState(false, false);
+
+            var missing = IsProofMissing(slotId, blockNumber, period);
+            return new ProofState(required, missing);
+        }
+
+        private bool IsProofRequired(byte[] slotId, ulong blockNumber)
+        {
+            var func = new IsProofRequiredFunction
+            {
+                Id = slotId
+            };
+            var result = gethNode.Call<IsProofRequiredFunction, IsProofRequiredOutputDTO>(Deployment.MarketplaceAddress, func, blockNumber);
+            return result.ReturnValue1;
+        }
+
+        private bool IsProofMissing(byte[] slotId, ulong blockNumber, ulong period)
+        {
+            try
+            {
+                var funcB = new MarkProofAsMissingFunction
+                {
+                    SlotId = slotId,
+                    Period = period
+                };
+                gethNode.Call<MarkProofAsMissingFunction>(Deployment.MarketplaceAddress, funcB, blockNumber);
+            }
+            catch (AggregateException exc)
+            {
+                if (exc.InnerExceptions.Count == 1)
+                {
+                    if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException))
+                    {
+                        return false;
+                    }
+                }
+                throw;
+            }
+            return true;
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,7 +1,6 @@
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Recipe;
-using Logging;
 
 namespace CodexContractsPlugin
 {

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,9 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -17,7 +17,8 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
-        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents();
+        ProofSubmittedEventDTO[] GetProofSubmittedEvents();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -86,12 +87,18 @@ namespace CodexContractsPlugin
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
-        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents()
         {
             var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
+        public ProofSubmittedEventDTO[] GetProofSubmittedEvents()
+        {
+            var events = gethNode.GetEvents<ProofSubmittedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
         private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
         {
             var result = e.Event;

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,11 +1,12 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexConvertors.Extensions;
-using NethereumWorkflow;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin
 {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,7 +1,7 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
-using GethPlugin;
-using NethereumWorkflow.BlockUtils;
+using BlockchainUtils;
 using Newtonsoft.Json;
+using Utils;
 
 namespace CodexContractsPlugin.Marketplace
 {
@@ -69,5 +69,11 @@ namespace CodexContractsPlugin.Marketplace
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
+
+    public partial class ProofSubmittedEventDTO : IHasBlock
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
+        private const string OpenApiYamlHash = "00-7D-C3-0B-D2-23-D6-6C-CA-C2-43-D0-9B-B4-63-FC-4F-FE-23-9F-B8-82-5F-3B-3F-6B-4F-1F-11-E9-48-16";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -82,12 +82,12 @@ namespace CodexPlugin
         private void OverwriteOpenApiYaml(string containerApi)
         {
             Log("API compatibility check failed. Updating CodexPlugin...");
-            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
-            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexClient/openapi.yaml. Expected: " + openApiFilePath);
 
             File.Delete(openApiFilePath);
             File.WriteAllText(openApiFilePath, containerApi);
-            Log("CodexPlugin/openapi.yaml has been updated.");
+            Log("CodexClient/openapi.yaml has been updated.");
         }
 
         private string Hash(string file)

ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs

@@ -0,0 +1,156 @@
+using CodexClient;
+using Core;
+using Utils;
+using System.Diagnostics;
+
+namespace CodexPlugin
+{
+    public class BinaryCodexStarter : ICodexStarter
+    {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly static NumberSource numberSource = new NumberSource(1);
+        private readonly static FreePortFinder freePortFinder = new FreePortFinder();
+        private readonly static object _lock = new object();
+        private readonly static string dataParentDir = "codex_disttest_datadirs";
+        private readonly static CodexExePath codexExePath = new CodexExePath();
+
+        static BinaryCodexStarter()
+        {
+            StopAllCodexProcesses();
+            DeleteParentDataDir();
+        }
+
+        public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+        }
+
+        public ICodexInstance[] BringOnline(CodexSetup codexSetup)
+        {
+            lock (_lock)
+            {
+                LogSeparator();
+                Log($"Starting {codexSetup.Describe()}...");
+
+                return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes);
+            }
+        }
+
+        public void Decommission()
+        {
+            lock (_lock)
+            {
+                processControlMap.StopAll();
+            }
+        }
+
+        private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes)
+        {
+            var result = new List<ICodexInstance>();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(StartBinary(startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        private ICodexInstance StartBinary(CodexStartupConfig config)
+        {
+            var name = GetName(config);
+            var dataDir = Path.Combine(dataParentDir, $"datadir_{numberSource.GetNextNumber()}");
+            var pconfig = new CodexProcessConfig(name, freePortFinder, dataDir);
+            Log(pconfig);
+
+            var factory = new CodexProcessRecipe(pconfig, codexExePath);
+            var recipe = factory.Initialize(config);
+
+            var startInfo = new ProcessStartInfo(
+                fileName: recipe.Cmd,
+                arguments: recipe.Args
+            );
+            //startInfo.UseShellExecute = true;
+            startInfo.RedirectStandardOutput = true;
+            startInfo.RedirectStandardError = true;
+
+            var process = Process.Start(startInfo);
+            if (process == null || process.HasExited)
+            {
+                throw new Exception("Failed to start");
+            }
+
+            var local = "localhost";
+            var instance = new CodexInstance(
+                name: name,
+                imageName: "binary",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: new Address("Disc", pconfig.LocalIpAddrs.ToString(), pconfig.DiscPort),
+                apiEndpoint: new Address("Api", "http://" + local, pconfig.ApiPort),
+                listenEndpoint: new Address("Listen", local, pconfig.ListenPort),
+                ethAccount: null,
+                metricsEndpoint: null
+            );
+
+            var pc = new BinaryProcessControl(pluginTools.GetLog(), process, pconfig);
+            processControlMap.Add(instance, pc);
+
+            return instance;
+        }
+
+        private string GetName(CodexStartupConfig config)
+        {
+            if (!string.IsNullOrEmpty(config.NameOverride))
+            {
+                return config.NameOverride + "_" + numberSource.GetNextNumber();
+            }
+            return "codex_" + numberSource.GetNextNumber();
+        }
+
+        private void LogSeparator()
+        {
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void Log(CodexProcessConfig pconfig)
+        {
+            Log(
+                "NodeConfig:Name=" + pconfig.Name +
+                "ApiPort=" + pconfig.ApiPort +
+                "DiscPort=" + pconfig.DiscPort +
+                "ListenPort=" + pconfig.ListenPort +
+                "DataDir=" + pconfig.DataDir
+            );
+        }
+
+        private void Log(string message)
+        {
+            pluginTools.GetLog().Log(message);
+        }
+
+        private static void DeleteParentDataDir()
+        {
+            if (Directory.Exists(dataParentDir))
+            {
+                Directory.Delete(dataParentDir, true);
+            }
+        }
+
+        private static void StopAllCodexProcesses()
+        {
+            var processes = Process.GetProcesses();
+            var codexes = processes.Where(p =>
+                p.ProcessName.ToLowerInvariant() == "codex" &&
+                p.MainModule != null &&
+                p.MainModule.FileName == codexExePath.Get()
+            ).ToArray();
+
+            foreach (var c in codexes)
+            {
+                c.Kill();
+                c.WaitForExit();
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/BinaryProcessControl.cs

@@ -0,0 +1,93 @@
+using System.Diagnostics;
+using CodexClient;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class BinaryProcessControl : IProcessControl
+    {
+        private readonly LogFile logFile;
+        private readonly Process process;
+        private readonly CodexProcessConfig config;
+        private List<string> logBuffer = new List<string>();
+        private readonly object bufferLock = new object();
+        private readonly List<Task> streamTasks = new List<Task>();
+        private bool running;
+
+        public BinaryProcessControl(ILog log, Process process, CodexProcessConfig config)
+        {
+            logFile = log.CreateSubfile(config.Name);
+
+            running = true;
+            this.process = process;
+            this.config = config;
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardOutput)));
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardError)));
+            streamTasks.Add(Task.Run(() => WriteLog()));
+        }
+
+        private void ReadProcessStream(StreamReader reader)
+        {
+            while (running)
+            {
+                var line = reader.ReadLine();
+                if (!string.IsNullOrEmpty(line))
+                {
+                    lock (bufferLock)
+                    {
+                        logBuffer.Add(line);
+                    }
+                }
+            }
+        }
+
+        private void WriteLog()
+        {
+            while (running || logBuffer.Count > 0)
+            {
+                if (logBuffer.Count > 0)
+                {
+                    List<string> lines = null!;
+                    lock (bufferLock)
+                    {
+                        lines = logBuffer;
+                        logBuffer = new List<string>();
+                    }
+
+                    logFile.WriteRawMany(lines);
+                }
+                else Thread.Sleep(100);
+            }
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            if (!Directory.Exists(config.DataDir)) throw new Exception("datadir not found");
+            Directory.Delete(config.DataDir, true);
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            return new DownloadedLog(logFile, config.Name);
+        }
+
+        public bool HasCrashed()
+        {
+            return process.HasExited;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            running = false;
+            process.Kill();
+
+            if (waitTillStopped)
+            {
+                process.WaitForExit();
+                foreach (var t in streamTasks) t.Wait();
+            }
+
+            DeleteDataDirFolder();
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs

@@ -0,0 +1,70 @@
+using CodexClient;
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class CodexContainerProcessControl : IProcessControl
+    {
+        private readonly IPluginTools tools;
+        private readonly RunningPod pod;
+        private readonly Action onStop;
+        private readonly ContainerCrashWatcher crashWatcher;
+
+        public CodexContainerProcessControl(IPluginTools tools, RunningPod pod, Action onStop)
+        {
+            this.tools = tools;
+            this.pod = pod;
+            this.onStop = onStop;
+
+            crashWatcher = tools.CreateWorkflow().CreateCrashWatcher(pod.Containers.Single());
+            crashWatcher.Start();
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            crashWatcher.Stop();
+            var workflow = tools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            onStop();
+            Log("Stopped.");
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            var workflow = tools.CreateWorkflow();
+            return workflow.DownloadContainerLog(pod.Containers.Single());
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            var container = pod.Containers.Single();
+
+            try
+            {
+                var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
+                var dataDir = dataDirVar.Value;
+                var workflow = tools.CreateWorkflow();
+                workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
+                Log("Deleted repo folder.");
+            }
+            catch (Exception e)
+            {
+                Log("Unable to delete repo folder: " + e);
+            }
+        }
+
+        public bool HasCrashed()
+        {
+            return crashWatcher.HasCrashed();
+        }
+
+        private void Log(string message)
+        {
+            tools.GetLog().Log(message);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using CodexClient;
+using CodexContractsPlugin;
 using GethPlugin;
 using KubernetesWorkflow.Types;
 
@@ -9,7 +10,7 @@ namespace CodexPlugin
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
             CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
             RunningPod? discordBotContainer, DeploymentMetadata metadata,
-            String id)
+            string id)
         {
             Id = id;
             CodexInstances = codexInstances;
@@ -20,7 +21,7 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
-        public String Id { get; }
+        public string Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
@@ -29,18 +30,6 @@ namespace CodexPlugin
         public DeploymentMetadata Metadata { get; }
     }
 
-    public class CodexInstance
-    {
-        public CodexInstance(RunningPod pod, DebugInfo info)
-        {
-            Pod = pod;
-            Info = info;
-        }
-
-        public RunningPod Pod { get; }
-        public DebugInfo Info { get; }
-    }
-
     public class DeploymentMetadata
     {
         public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,

ProjectPlugins/CodexPlugin/CodexExePath.cs

@@ -0,0 +1,29 @@
+namespace CodexPlugin
+{
+    public class CodexExePath
+    {
+        private readonly string[] paths = [
+            Path.Combine("d:", "Dev", "nim-codex", "build", "codex.exe"),
+            Path.Combine("c:", "Projects", "nim-codex", "build", "codex.exe")
+        ];
+
+        private string selectedPath = string.Empty;
+
+        public CodexExePath()
+        {
+            foreach (var p in paths)
+            {
+                if (File.Exists(p))
+                {
+                    selectedPath = p;
+                    return;
+                }
+            }
+        }
+
+        public string Get()
+        {
+            return selectedPath;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs

@@ -0,0 +1,46 @@
+using CodexClient;
+using KubernetesWorkflow.Types;
+using Utils;
+
+namespace CodexPlugin
+{
+    public static class CodexInstanceContainerExtension
+    {
+        public static ICodexInstance CreateFromPod(RunningPod pod)
+        {
+            var container = pod.Containers.Single();
+
+            return new CodexInstance(
+                name: container.Name,
+                imageName: container.Recipe.Image,
+                startUtc: container.Recipe.RecipeCreatedUtc,
+                discoveryEndpoint: SetClusterInternalIpAddress(pod, container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag)),
+                apiEndpoint: container.GetAddress(CodexContainerRecipe.ApiPortTag),
+                listenEndpoint: container.GetInternalAddress(CodexContainerRecipe.ListenPortTag),
+                ethAccount: container.Recipe.Additionals.Get<EthAccount>(),
+                metricsEndpoint: GetMetricsEndpoint(container)
+            );
+        }
+
+        private static Address SetClusterInternalIpAddress(RunningPod pod, Address address)
+        {
+            return new Address(
+                logName: address.LogName,
+                host: pod.PodInfo.Ip,
+                port: address.Port
+            );
+        }
+
+        private static Address? GetMetricsEndpoint(RunningContainer container)
+        {
+            try
+            {
+                return container.GetInternalAddress(CodexContainerRecipe.MetricsPortTag);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,53 +0,0 @@
-using CodexPlugin.Hooks;
-using Core;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
-
-namespace CodexPlugin
-{
-    public interface ICodexNodeFactory
-    {
-        CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
-        CrashWatcher CreateCrashWatcher(RunningContainer c);
-    }
-
-    public class CodexNodeFactory : ICodexNodeFactory
-    {
-        private readonly IPluginTools tools;
-        private readonly CodexHooksFactory codexHooksFactory;
-
-        public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
-        {
-            this.tools = tools;
-            this.codexHooksFactory = codexHooksFactory;
-        }
-
-        public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        {
-            var ethAccount = GetEthAccount(access);
-            var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
-
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount, hooks);
-            return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
-        }
-
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount, ICodexNodeHooks hooks)
-        {
-            if (ethAccount == null) return new MarketplaceUnavailable();
-            return new MarketplaceAccess(tools.GetLog(), codexAccess, hooks);
-        }
-
-        private EthAccount? GetEthAccount(CodexAccess access)
-        {
-            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
-            if (ethAccount == null) return null;
-            return ethAccount;
-        }
-
-        public CrashWatcher CreateCrashWatcher(RunningContainer c)
-        {
-            return tools.CreateWorkflow().CreateCrashWatcher(c);
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,25 +1,23 @@
-using Core;
-using KubernetesWorkflow.Types;
-using MetricsPlugin;
+using CodexClient;
+using Core;
 using System.Collections;
+using Utils;
 
 namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
     {
-        void BringOffline(bool waitTillStopped);
+        void Stop(bool waitTillStopped);
         ICodexNode this[int index] { get; }
     }
 
     public class CodexNodeGroup : ICodexNodeGroup
     {
-        private readonly CodexStarter starter;
+        private readonly ICodexNode[] nodes;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(IPluginTools tools, ICodexNode[] nodes)
         {
-            this.starter = starter;
-            Containers = containers;
-            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            this.nodes = nodes;
             Version = new DebugInfoVersion();
         }
 
@@ -31,25 +29,23 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline(bool waitTillStopped)
+        public void Stop(bool waitTillStopped)
         {
-            starter.BringOffline(this, waitTillStopped);
-            // Clear everything. Prevent accidental use.
-            Nodes = Array.Empty<CodexNode>();
-            Containers = null!;
+            foreach (var node in Nodes) node.Stop(waitTillStopped);
         }
 
         public void Stop(CodexNode node, bool waitTillStopped)
         {
-            starter.Stop(node.Pod, waitTillStopped);
-            Nodes = Nodes.Where(n => n != node).ToArray();
-            Containers = Containers.Where(c => c != node.Pod).ToArray();
+            node.Stop(waitTillStopped);
         }
 
-        public RunningPod[] Containers { get; private set; }
-        public CodexNode[] Nodes { get; private set; }
+        public ICodexNode[] Nodes => nodes;
         public DebugInfoVersion Version { get; private set; }
-        public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
+
+        public Address[] GetMetricsScrapeTargets()
+        {
+            return Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray();
+        }
 
         public IEnumerator<ICodexNode> GetEnumerator()
         {
@@ -63,12 +59,11 @@ namespace CodexPlugin
 
         public string Describe()
         {
-            return $"group:[{Containers.Describe()}]";
+            return $"group:[{string.Join(",", Nodes.Select(n => n.GetName()))}]";
         }
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
@@ -79,16 +74,6 @@ namespace CodexPlugin
             }
 
             Version = first;
-            foreach (var node in Nodes) node.Initialize();
-        }
-
-        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
-        {
-            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
-            var access = new CodexAccess(tools, c, watcher);
-            var node = factory.CreateOnlineCodexNode(access, this);
-            node.Awake();
-            return node;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,48 +1,68 @@
-using CodexPlugin.Hooks;
+using CodexClient;
+using CodexClient.Hooks;
 using Core;
-using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public class CodexPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
-        private readonly CodexStarter codexStarter;
+        private const bool UseContainers = true;
+
+        private readonly ICodexStarter codexStarter;
         private readonly IPluginTools tools;
         private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace;
+        private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory();
+        private readonly ProcessControlMap processControlMap = new ProcessControlMap();
+        private readonly CodexWrapper codexWrapper;
 
         public CodexPlugin(IPluginTools tools)
         {
-            codexStarter = new CodexStarter(tools);
             this.tools = tools;
+
+            codexStarter = CreateCodexStarter();
+            codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory);
+        }
+
+        private ICodexStarter CreateCodexStarter()
+        {
+            if (UseContainers)
+            {
+                Log("Using Containerized Codex instances");
+                return new ContainerCodexStarter(tools, processControlMap);
+            }
+
+            Log("Using Binary Codex instances");
+            return new BinaryCodexStarter(tools, processControlMap);
         }
 
         public string LogPrefix => "(Codex) ";
 
         public void Announce()
         {
-            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
-            metadata.Add("codexid", codexStarter.GetCodexId());
-            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
+            metadata.Add("codexid", codexWrapper.GetCodexId());
+            metadata.Add("codexrevision", codexWrapper.GetCodexRevision());
         }
 
         public void Decommission()
         {
+            codexStarter.Decommission();
         }
 
-        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexInstance[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(ICodexInstance[] instances)
         {
-            containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
-            return codexStarter.WrapCodexContainers(coreInterface, containers);
+            instances = instances.Select(c => SerializeGate.Gate(c as CodexInstance)).ToArray();
+            return codexWrapper.WrapCodexInstances(instances);
         }
 
         public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
@@ -62,9 +82,10 @@ namespace CodexPlugin
             }
         }
 
-        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider)
         {
-            codexStarter.HooksFactory.Provider = hooksProvider;
+            if (hooksFactory.Providers.Contains(hooksProvider)) return;
+            hooksFactory.Providers.Add(hooksProvider);
         }
 
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -6,14 +6,6 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <ItemGroup>
-    <None Remove="openapi.yaml" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
       <PrivateAssets>all</PrivateAssets>
@@ -30,6 +22,7 @@
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
+    <ProjectReference Include="..\CodexClient\CodexClient.csproj" />
     <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />

ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs

@@ -0,0 +1,160 @@
+using System.Net.Sockets;
+using System.Net;
+using Nethereum.Util;
+
+namespace CodexPlugin
+{
+    public class ProcessRecipe
+    {
+        public ProcessRecipe(string cmd, string[] args)
+        {
+            Cmd = cmd;
+            Args = args;
+        }
+
+        public string Cmd { get; }
+        public string[] Args { get; }
+    }
+
+    public class CodexProcessConfig
+    {
+        public CodexProcessConfig(string name, FreePortFinder freePortFinder, string dataDir)
+        {
+            ApiPort = freePortFinder.GetNextFreePort();
+            DiscPort = freePortFinder.GetNextFreePort();
+            ListenPort = freePortFinder.GetNextFreePort();
+            Name = name;
+            DataDir = dataDir;
+            var host = Dns.GetHostEntry(Dns.GetHostName());
+            var addrs = host.AddressList.Where(a => a.AddressFamily == AddressFamily.InterNetwork).ToList();
+
+            LocalIpAddrs = addrs.First();
+        }
+
+        public int ApiPort { get; }
+        public int DiscPort { get; }
+        public int ListenPort { get; }
+        public string Name { get; }
+        public string DataDir { get; }
+        public IPAddress LocalIpAddrs { get; }
+    }
+
+    public class CodexProcessRecipe
+    {
+        private readonly CodexProcessConfig pc;
+        private readonly CodexExePath codexExePath;
+
+        public CodexProcessRecipe(CodexProcessConfig pc, CodexExePath codexExePath)
+        {
+            this.pc = pc;
+            this.codexExePath = codexExePath;
+        }
+
+        public ProcessRecipe Initialize(CodexStartupConfig config)
+        {
+            args.Clear();
+            
+            AddArg("--api-port", pc.ApiPort);
+            AddArg("--api-bindaddr", "0.0.0.0");
+
+            AddArg("--data-dir", pc.DataDir);
+
+            AddArg("--disc-port", pc.DiscPort);
+            AddArg("--log-level", config.LogLevelWithTopics());
+
+            // This makes the node announce itself to its local IP address.
+            AddArg("--nat", $"extip:{pc.LocalIpAddrs.ToStringInvariant()}");
+            
+            AddArg("--listen-addrs", $"/ip4/0.0.0.0/tcp/{pc.ListenPort}");
+
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddArg("--bootstrap-node", config.BootstrapSpr);
+            }
+            if (config.StorageQuota != null)
+            {
+                AddArg("--storage-quota", config.StorageQuota.SizeInBytes.ToString()!);
+            }
+            if (config.BlockTTL != null)
+            {
+                AddArg("--block-ttl", config.BlockTTL.ToString()!);
+            }
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddArg("--block-mi", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddArg("--block-mn", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsEnabled)
+            {
+                throw new Exception("Not supported");
+                //var metricsPort = CreateApiPort(config, MetricsPortTag);
+                //AddEnvVar("CODEX_METRICS", "true");
+                //AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
+                //AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                //AddPodAnnotation("prometheus.io/scrape", "true");
+                //AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
+            }
+
+            if (config.SimulateProofFailures != null)
+            {
+                throw new Exception("Not supported");
+                //AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+
+            if (config.MarketplaceConfig != null)
+            {
+                throw new Exception("Not supported");
+                //var mconfig = config.MarketplaceConfig;
+                //var gethStart = mconfig.GethNode.StartResult;
+                //var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
+                //var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
+
+                //AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
+                //AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
+
+                //var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
+
+                //// Custom scripting in the Codex test image will write this variable to a private-key file,
+                //// and pass the correct filename to Codex.
+                //var account = marketplaceSetup.EthAccountSetup.GetNew();
+                //AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
+                //Additional(account);
+
+                //SetCommandOverride(marketplaceSetup);
+                //if (marketplaceSetup.IsValidator)
+                //{
+                //    AddEnvVar("CODEX_VALIDATOR", "true");
+                //}
+            }
+
+            //if (!string.IsNullOrEmpty(config.NameOverride))
+            //{
+            //    AddEnvVar("CODEX_NODENAME", config.NameOverride);
+            //}
+
+            return Create();
+        }
+
+        private ProcessRecipe Create()
+        {
+            return new ProcessRecipe(
+                cmd: codexExePath.Get(),
+                args: args.ToArray());
+        }
+
+        private readonly List<string> args = new List<string>();
+
+        private void AddArg(string arg, string val)
+        {
+            args.Add($"{arg}={val}");
+        }
+
+        private void AddArg(string arg, int val)
+        {
+            args.Add($"{arg}={val}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using CodexClient;
+using CodexContractsPlugin;
 using GethPlugin;
 using KubernetesWorkflow;
 using Utils;
@@ -53,6 +54,7 @@ namespace CodexPlugin
         public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
         public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn;
+        public CodexLogLevel MarketplaceInfra { get; set; } = CodexLogLevel.Warn;
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -223,7 +225,7 @@ namespace CodexPlugin
         {
             if (pinned) return accounts.Last();
 
-            var a = EthAccount.GenerateNew();
+            var a = EthAccountGenerator.GenerateNew();
             accounts.Add(a);
             return a;
         }

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexClient;
+using KubernetesWorkflow;
 using Utils;
 
 namespace CodexPlugin
@@ -29,6 +30,7 @@ namespace CodexPlugin
                 {
                     "discv5",
                     "providers",
+                    "routingtable",
                     "manager",
                     "cache",
                 };
@@ -79,12 +81,20 @@ namespace CodexPlugin
                     "json",
                     "serialization"
                 };
+                var marketplaceInfraTopics = new[]
+                {
+                    "JSONRPC-WS-CLIENT",
+                    "JSONRPC-HTTP-CLIENT",
+                    "codex",
+                    "repostore"
+                };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
                     $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
-                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)}";
+                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)};" +
+                    $"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/CodexWrapper.cs

@@ -0,0 +1,80 @@
+using CodexClient;
+using CodexClient.Hooks;
+using Core;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class CodexWrapper
+    {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly CodexHooksFactory hooksFactory;
+        private DebugInfoVersion? versionResponse;
+
+        public CodexWrapper(IPluginTools pluginTools, ProcessControlMap processControlMap, CodexHooksFactory hooksFactory)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+            this.hooksFactory = hooksFactory;
+        }
+
+        public string GetCodexId()
+        {
+            if (versionResponse != null) return versionResponse.Version;
+            return "unknown";
+        }
+
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.Revision;
+            return "unknown";
+        }
+
+        public ICodexNodeGroup WrapCodexInstances(ICodexInstance[] instances)
+        {
+            var codexNodeFactory = new CodexNodeFactory(
+                log: pluginTools.GetLog(),
+                fileManager: pluginTools.GetFileManager(),
+                hooksFactory: hooksFactory,
+                httpFactory: pluginTools,
+                processControlFactory: processControlMap);
+
+            var group = CreateCodexGroup(instances, codexNodeFactory);
+
+            pluginTools.GetLog().Log($"Codex version: {group.Version}");
+            versionResponse = group.Version;
+
+            return group;
+        }
+
+        private CodexNodeGroup CreateCodexGroup(ICodexInstance[] instances, CodexNodeFactory codexNodeFactory)
+        {
+            var nodes = instances.Select(codexNodeFactory.CreateCodexNode).ToArray();
+            var group = new CodexNodeGroup(pluginTools, nodes);
+
+            try
+            {
+                Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
+            }
+            catch
+            {
+                CodexNodesNotOnline(instances);
+                throw;
+            }
+
+            return group;
+        }
+
+        private void CodexNodesNotOnline(ICodexInstance[] instances)
+        {
+            pluginTools.GetLog().Log("Codex nodes failed to start");
+            var log = pluginTools.GetLog();
+            foreach (var i in instances)
+            {
+                var pc = processControlMap.Get(i);
+                pc.DownloadLog(log.CreateSubfile(i.Name + "_failed_to_start"));
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs

@@ -1,29 +1,26 @@
-using CodexPlugin.Hooks;
+using CodexClient;
 using Core;
-using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
-using Logging;
+using Utils;
 
 namespace CodexPlugin
 {
-    public class CodexStarter
+    public class ContainerCodexStarter : ICodexStarter
     {
         private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
         private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
         private readonly ApiChecker apiChecker;
-        private DebugInfoVersion? versionResponse;
 
-        public CodexStarter(IPluginTools pluginTools)
+        public ContainerCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
         {
             this.pluginTools = pluginTools;
-
+            this.processControlMap = processControlMap;
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public CodexHooksFactory HooksFactory { get; } = new CodexHooksFactory();
-
-        public RunningPod[] BringOnline(CodexSetup codexSetup)
+        public ICodexInstance[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -43,51 +40,11 @@ namespace CodexPlugin
             }
             LogSeparator();
 
-            return containers;
+            return containers.Select(CreateInstance).ToArray();
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public void Decommission()
         {
-            var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
-
-            var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);
-
-            Log($"Codex version: {group.Version}");
-            versionResponse = group.Version;
-
-            return group;
-        }
-
-        public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
-        {
-            Log($"Stopping {group.Describe()}...");
-            StopCrashWatcher(group);
-            var workflow = pluginTools.CreateWorkflow();
-            foreach (var c in group.Containers)
-            {
-                workflow.Stop(c, waitTillStopped);
-            }
-            Log("Stopped.");
-        }
-
-        public void Stop(RunningPod pod, bool waitTillStopped)
-        {
-            Log($"Stopping node...");
-            var workflow = pluginTools.CreateWorkflow();
-            workflow.Stop(pod, waitTillStopped);
-            Log("Stopped.");
-        }
-
-        public string GetCodexId()
-        {
-            if (versionResponse != null) return versionResponse.Version;
-            return recipe.Image;
-        }
-
-        public string GetCodexRevision()
-        {
-            if (versionResponse != null) return versionResponse.Revision;
-            return "unknown";
         }
 
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
@@ -118,27 +75,15 @@ namespace CodexPlugin
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private ICodexInstance CreateInstance(RunningPod pod)
         {
-            var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
-
-            try
+            var instance = CodexInstanceContainerExtension.CreateFromPod(pod);
+            var processControl = new CodexContainerProcessControl(pluginTools, pod, onStop: () =>
             {
-                Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
-            }
-            catch
-            {
-                CodexNodesNotOnline(coreInterface, runningContainers);
-                throw;
-            }
-
-            return group;
-        }
-
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
-        {
-            Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
+                processControlMap.Remove(instance);
+            });
+            processControlMap.Add(instance, processControl);
+            return instance;
         }
 
         private void LogSeparator()
@@ -157,13 +102,5 @@ namespace CodexPlugin
         {
             pluginTools.GetLog().Log(message);
         }
-
-        private void StopCrashWatcher(CodexNodeGroup group)
-        {
-            foreach (var node in group)
-            {
-                node.CrashWatcher.Stop();
-            }
-        }
     }
 }

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -1,19 +1,19 @@
-using CodexPlugin.Hooks;
+using CodexClient;
+using CodexClient.Hooks;
 using Core;
-using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static ICodexInstance[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, ICodexInstance[] instances)
         {
-            return Plugin(ci).WrapCodexContainers(ci, containers);
+            return Plugin(ci).WrapCodexContainers(instances);
         }
 
         public static ICodexNode StartCodexNode(this CoreInterface ci)
@@ -39,9 +39,9 @@ namespace CodexPlugin
             return ci.StartCodexNodes(number, s => { });
         }
 
-        public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
+        public static void AddCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
         {
-            Plugin(ci).SetCodexHooksProvider(hooksProvider);
+            Plugin(ci).AddCodexHooksProvider(hooksProvider);
         }
 
         private static CodexPlugin Plugin(CoreInterface ci)

ProjectPlugins/CodexPlugin/FreePortFinder.cs

@@ -0,0 +1,43 @@
+using System.Net.NetworkInformation;
+
+namespace CodexPlugin
+{
+    public class FreePortFinder
+    {
+        private readonly object _lock = new object();
+        private int nextPort = 8080;
+
+        public int GetNextFreePort()
+        {
+            lock (_lock)
+            {
+                return Next();
+            }
+        }
+
+        private int Next()
+        {
+            while (true)
+            {
+                var p = nextPort;
+                nextPort++;
+
+                if (!IsInUse(p))
+                {
+                    return p;
+                }
+
+                if (nextPort > 30000) throw new Exception("Running out of ports.");
+            }
+        }
+
+        private bool IsInUse(int port)
+        {
+            var ipProps = IPGlobalProperties.GetIPGlobalProperties();
+            if (ipProps.GetActiveTcpConnections().Any(t => t.LocalEndPoint.Port == port)) return true;
+            if (ipProps.GetActiveTcpListeners().Any(t => t.Port == port)) return true;
+            if (ipProps.GetActiveUdpListeners().Any(u => u.Port == port)) return true;
+            return false;
+        }
+    }
+}