investigate wip

* Fix job status check

* Add a delay before get Kubernetes job status

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,121 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +19,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -6,6 +6,7 @@ namespace KubernetesWorkflow
     {
         string ContainerName { get; }
 
+        void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
@@ -25,58 +26,39 @@ namespace KubernetesWorkflow
 
         public string ContainerName { get; }
 
-        public void IterateLines(Action<string> action, params string[] thatContain)
+        public void IterateLines(Action<string> action)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
             while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
             {
                 if (thatContain.All(line.Contains))
                 {
                     action(line);
                 }
-                line = streamReader.ReadLine();
-            }
+            });
         }
 
         public string[] GetLinesContaining(string expectedString)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
+            return FindLinesThatContain([expectedString]);
         }
 
         public string[] FindLinesThatContain(params string[] tags)
         {
             var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
+            IterateLines(result.Add, tags);
             return result.ToArray();
         }
 

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,6 +2,7 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +25,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        void WaitUntilNextPeriod();
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -114,6 +116,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -4,6 +4,7 @@ using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -75,9 +77,20 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
         }
 
         private void EnsureCompatbility(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -23,9 +23,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +110,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -32,6 +32,22 @@ namespace CodexPlugin
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
@@ -80,6 +96,18 @@ namespace CodexPlugin
             return fileResponse.Stream;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
         public LocalDatasetList LocalFiles()
         {
             return mapper.Map(OnCodex(api => api.ListDataAsync()));

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -14,13 +14,16 @@ namespace CodexPlugin
     {
         string GetName();
         string GetPeerId();
-        DebugInfo GetDebugInfo();
+        DebugInfo GetDebugInfo(bool log = false);
+        string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
         ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
@@ -120,14 +123,22 @@ namespace CodexPlugin
             return peerId;
         }
 
-        public DebugInfo GetDebugInfo()
+        public DebugInfo GetDebugInfo(bool log = false)
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
             return debugInfo;
         }
 
+        public string GetSpr()
+        {
+            return CodexAccess.GetSpr();
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
@@ -192,6 +203,16 @@ namespace CodexPlugin
             return file;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            return CodexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            return CodexAccess.DownloadManifestOnly(cid);
+        }
+
         public LocalDatasetList LocalFiles()
         {
             return CodexAccess.LocalFiles();

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -20,7 +20,7 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
@@ -55,6 +55,10 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
@@ -70,5 +74,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,7 +1,5 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Logging;
-using System.Data;
 using Utils;
 
 namespace CodexPlugin

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,4 +1,6 @@
-using CodexPlugin.Hooks;
+using CodexContractsPlugin;
+using CodexPlugin.Hooks;
+using GethPlugin;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -12,7 +14,7 @@ namespace CodexPlugin
         ContentId ContentId { get; }
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished();
+        void WaitForStorageContractFinished(ICodexContracts contracts);
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -62,7 +64,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished()
+        public void WaitForStorageContractFinished(ICodexContracts contracts)
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -74,6 +76,13 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+
+            contracts.WaitUntilNextPeriod();
+            contracts.WaitUntilNextPeriod();
+
+            var blocks = 3;
+            Log($"Waiting {blocks} blocks for nodes to process payouts...");
+            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
         }
 
         public StoragePurchase GetPurchaseStatus(string purchaseId)

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace GethPlugin
     public class GethContainerRecipe : ContainerRecipeFactory
     {
         public static string DockerImage { get; } = "codexstorage/dist-tests-geth:latest";
+        public static TimeSpan BlockInterval { get; } = TimeSpan.FromSeconds(1.0);
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
         public const string HttpPortTag = "http_port";

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -14,8 +14,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -14,8 +14,8 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexReleaseTests/CodexReleaseTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.3" />
+	<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+	<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs

@@ -1,26 +1,16 @@
 using CodexPlugin;
+using CodexTests;
 using FileUtils;
 using NUnit.Framework;
 using System.Diagnostics;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
-    [TestFixture]
-    public class OneClientTests : CodexDistTest
+    public class InterruptUploadTest : CodexDistTest
     {
         [Test]
-        public void OneClientTest()
-        {
-            var node = StartCodex();
-
-            PerformOneClientTest(node);
-
-            LogNodeStatus(node);
-        }
-
-        [Test]
-        public void InterruptUploadTest()
+        public void UploadInterruptTest()
         {
             var nodes = StartCodex(10);
 
@@ -28,6 +18,8 @@ namespace CodexTests.BasicTests
             Task.WaitAll(tasks.ToArray());
 
             Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
         }
 
         private bool RunInterruptUploadTest(ICodexNode node)
@@ -51,16 +43,5 @@ namespace CodexTests.BasicTests
             var filePath = file.Filename;
             return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
         }
-
-        private void PerformOneClientTest(ICodexNode primary)
-        {
-            var testFile = GenerateTestFile(1.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = primary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
     }
 }

Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs

@@ -0,0 +1,33 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class ManifestOnlyDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void ManifestOnlyTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(2.GB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var localDataset = downloader.DownloadManifestOnly(cid);
+
+            Thread.Sleep(1000);
+
+            var spaceDiff = startSpace.FreeBytes - downloader.Space().FreeBytes;
+
+            Assert.That(spaceDiff, Is.LessThan(64.KB().SizeInBytes));
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/OneClientTest.cs

@@ -0,0 +1,39 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class OneClientTest : CodexDistTest
+    {
+        [Test]
+        public void OneClient()
+        {
+            var node = StartCodex();
+
+            PerformOneClientTest(node);
+
+            LogNodeStatus(node);
+        }
+
+        private void PerformOneClientTest(ICodexNode primary)
+        {
+            var testFile = GenerateTestFile(1.MB());
+
+            var contentId = primary.UploadFile(testFile);
+
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = primary.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs

@@ -0,0 +1,47 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class StreamlessDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void StreamlessTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(10.MB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var start = DateTime.UtcNow;
+            var localDataset = downloader.DownloadStreamless(cid);
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+
+            // TODO: We have no way to inspect the status or progress of the download.
+            // We use local space information to estimate.
+            var retry = new Retry("Checking local space",
+                maxTimeout: TimeSpan.FromMinutes(2),
+                sleepAfterFail: TimeSpan.FromSeconds(3),
+                onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = downloader.Space();
+                var expected = startSpace.FreeBytes - size;
+                if (space.FreeBytes > expected) throw new Exception("Expected free space not reached.");
+            });
+
+            // Stop the uploader node and verify that the downloader has the data.
+            uploader.Stop(waitTillStopped: true);
+            var downloaded = downloader.DownloadContent(cid);
+            file.AssertIsEqual(downloaded);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/SwarmTest.cs

@@ -0,0 +1,198 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class SwarmTests : CodexDistTest
+    {
+        [Test]
+        [Ignore("a")]
+        [Combinatorial]
+        public void SmallSwarm(
+            [Values(true, false)] bool peerImage,
+            [Values(2, 5, 10)] int NumberOfNodes,
+            [Values(2, 5, 10)] int FileSizeMb
+        )
+        {
+            // "thatbenbierens/nim-codex:peerselect2"
+            // "codexstorage/nim-codex:latest-dist-tests"
+
+            if (peerImage)
+            {
+                CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:peerselect2";
+            }
+            else
+            {
+                CodexContainerRecipe.DockerImageOverride = "codexstorage/nim-codex:0.1.9-dist-tests";
+            }
+
+            var boot = StartCodex(s => s.WithName("Bootstrap"));
+
+            var nodes = StartCodex(NumberOfNodes, s => s.WithBootstrapNode(boot));
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, FileSizeMb)).ToArray();
+
+            var tasks = ParallelDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesDownloadedCorrectly(files);
+        }
+
+        [Test]
+        [Combinatorial]
+        public void Spppeeeed(
+            [Values(
+            23
+            )] int idx
+        )
+        {
+            string[] imgs = [
+                            //"thatbenbierens/nim-codex:peerselect2", // S review comments
+            "thatbenbierens/nim-codex:peerselect3", // X send wrong wantblock
+            "thatbenbierens/nim-codex:peerselect4", // S send wrong wantblock
+            "thatbenbierens/nim-codex:peerselect5", // X old task handler
+            "thatbenbierens/nim-codex:peerselect6", // S track only wantblocks
+            "thatbenbierens/nim-codex:peerselect7", // S old request proc
+            "thatbenbierens/nim-codex:peerselect8", // F! 3607b88 - sends wantBlock to peers with block. wantHave to everyone else
+            
+            "thatbenbierens/nim-codex:peerselect9", // [6] F + Slowing! 64e691b - Fixes issue where peerWants are only stored for type wantBlock.
+            //[2024-12-05T15:38:32.3332175Z] DL(12 secs)
+            //[2024-12-05T15:38:58.6083189Z] DL(20 secs)
+            //[2024-12-05T15:39:36.2547136Z] DL(29 secs)
+            //[2024-12-05T15:40:27.0061893Z] DL(38 secs)
+            //[2024-12-05T15:41:27.8224210Z] DL(47 secs)
+
+            "thatbenbierens/nim-codex:peerselect9log2", // S [7] same, var schedule + checking peerwants list length
+            "thatbenbierens/nim-codex:peerselect9log7", // S but stable [8] same, var schedule + checking peerwants list length
+            "thatbenbierens/nim-codex:peerselect9log8", // S [9] patch for not storing cancels + ed3e91c - Review comments by Dmitriy
+            "thatbenbierens/nim-codex:peerselect9log9", // S [10] ref object
+            "thatbenbierens/nim-codex:peerselect9log10", // S [11] presencecheck only new wants
+            "thatbenbierens/nim-codex:peerselect9log11", // S [12] same but no metrics + trace "wantList.entries.len" == always 1
+            "thatbenbierens/nim-codex:peerselect9log12", // S [13] always schedule peer
+            "thatbenbierens/nim-codex:peerselect9log13", // F! [14] 64e691b + new entry add if not e.cancel
+            "thatbenbierens/nim-codex:peerselect9log14", // F [15] ed3e91c + proc wantListHandler from "64e691b + new entry add if not e.cancel"
+            "thatbenbierens/nim-codex:peerselect9log15", // F [16] same, move metrics up
+            "thatbenbierens/nim-codex:peerselect9log16", // F [17] 1f063fe (branchlatest) proc wantlisthandler from previous
+            "thatbenbierens/nim-codex:peerselect9log17", // F [18] prev + restore schedulePeer bool
+            "thatbenbierens/nim-codex:peerselect9log18", // S [19] newcommit + moves presence check behind !cancel + type = wantHave
+            "thatbenbierens/nim-codex:peerselect9log19", // S [20] newcommit + moves presence check behind !cancel
+
+            "thatbenbierens/nim-codex:peerselect9log20", // F! [21] newcommit + moves presence check behind type == wanthave
+            "thatbenbierens/nim-codex:peerselect9log22", // ? [22] same + logging + logging
+            "thatbenbierens/nim-codex:peerselect9log23", // ? [23] same + logging + logging intentionally broken to compare!
+
+
+
+            "codexstorage/nim-codex:0.1.9-dist-tests", // F
+            "codexstorage/nim-codex:sha-8e29939-dist-tests", // F 8e29939 - Send pluralized wantBlock messages (#1016)
+            "codexstorage/nim-codex:sha-2124996-dist-tests"  // F 2124996 - Requesting the same CID sometimes causes a worker to discard the request if it's already inflight by another worker. (#1002)
+                ];
+
+            var img = imgs[idx];
+
+            CodexContainerRecipe.DockerImageOverride = img;
+
+            var boot = StartCodex(s => s.WithName("Bootstrap"));
+            var uploader = StartCodex(s => s.WithName("Uploader").WithBootstrapNode(boot));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(boot));
+
+            var total = TimeSpan.Zero;
+            var number = 1;
+
+            for (var i = 0; i < number; i++)
+            {
+                var file = GenerateTestFile(100.MB());
+                var cid = uploader.UploadFile(file);
+
+                var duration = Stopwatch.Measure(GetTestLog(), "DL", () =>
+                {
+                    downloader.DownloadContent(cid);
+                }) ;
+                
+                total += duration;
+                if (duration.TotalMinutes > 1.0) Assert.Fail("too slow");
+            }
+
+            var avg = total / number;
+            Log($"{img} 100MB download average duration: {avg}");
+        }
+
+        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, int fileSizeMb)
+        {
+            var file = GenerateTestFile(fileSizeMb.MB());
+            var cid = node.UploadFile(file);
+            return new SwarmTestNetworkFile(file, cid);
+        }
+
+        private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                tasks.Add(StartDownload(node, files));
+            }
+
+            return tasks.ToArray();
+        }
+
+        private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
+        {
+            return Task.Run(() =>
+            {
+                var remaining = files.ToList();
+
+                while (remaining.Count > 0)
+                {
+                    var file = remaining.PickOneRandom();
+                    try
+                    {
+                        var dl = node.DownloadContent(file.Cid);
+                        lock (file.Lock)
+                        {
+                            file.Downloaded.Add(dl);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        file.Error = ex;
+                    }
+                }
+            });
+        }
+
+        private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
+        {
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                lock (file.Lock)
+                {
+                    foreach (var dl in file.Downloaded)
+                    {
+                        file.Original.AssertIsEqual(dl);
+                    }
+                }
+            }
+        }
+
+        private class SwarmTestNetworkFile
+        {
+            public SwarmTestNetworkFile(TrackedFile original, ContentId cid)
+            {
+                Original = original;
+                Cid = cid;
+            }
+
+            public TrackedFile Original { get; }
+            public ContentId Cid { get; }
+            public object Lock { get; } = new object();
+            public List<TrackedFile?> Downloaded { get; } = new List<TrackedFile?>();
+            public Exception? Error { get; set; } = null;
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs

@@ -0,0 +1,32 @@
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class ThreeClientTest : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ThreeClient()
+        {
+            var primary = StartCodex();
+            var secondary = StartCodex();
+
+            var testFile = GenerateTestFile(10.MB());
+
+            var contentId = primary.UploadFile(testFile);
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = secondary.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, primary, secondary);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/TwoClientTest.cs

@@ -1,8 +1,14 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
     [TestFixture]
     public class TwoClientTests : CodexDistTest
@@ -17,6 +23,7 @@ namespace CodexTests.BasicTests
         }
 
         [Test]
+        [Ignore("Location selection is currently unavailable.")]
         public void TwoClientsTwoLocationsTest()
         {
             var locations = Ci.GetKnownLocations();
@@ -42,8 +49,10 @@ namespace CodexTests.BasicTests
             var testFile = GenerateTestFile(size);
 
             var contentId = uploader.UploadFile(testFile);
+            AssertNodesContainFile(contentId, uploader);
 
             var downloadedFile = downloader.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, uploader, downloader);
 
             testFile.AssertIsEqual(downloadedFile);
             CheckLogForErrors(uploader, downloader);

Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs

@@ -0,0 +1,38 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class UnknownCidTest : CodexDistTest
+    {
+        [Test]
+        public void DownloadingUnknownCidDoesNotCauseCrash()
+        {
+            var node = StartCodex();
+
+            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
+
+            var localFiles = node.LocalFiles().Content;
+            CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid);
+
+            try
+            {
+                node.DownloadContent(unknownCid);
+            }
+            catch (Exception ex)
+            {
+                var expectedMessage = $"Download of '{unknownCid.Id}' timed out";
+                if (!ex.Message.StartsWith(expectedMessage)) throw;
+            }
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -0,0 +1,20 @@
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractFailedTest : CodexDistTest
+    {
+        [Test]
+        [Ignore("TODO - Test in which hosts are punished for failing a contract")]
+        public void ContractFailed()
+        {
+
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs

@@ -0,0 +1,18 @@
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractRepairedTest
+    {
+        [Test]
+        [Ignore("TODO - Test in which a host fails, but the slot is repaired")]
+        public void ContractRepaired()
+        {
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

@@ -0,0 +1,73 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class ContractSuccessfulTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 4;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        public void ContractSuccessful()
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            request.WaitForStorageContractFinished(GetContracts());
+
+            AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -0,0 +1,254 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using CodexTests;
+using DistTestCore;
+using GethPlugin;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
+    {
+        private readonly Dictionary<TestLifecycle, MarketplaceHandle> handles = new Dictionary<TestLifecycle, MarketplaceHandle>();
+        protected const int StartingBalanceTST = 1000;
+        protected const int StartingBalanceEth = 10;
+
+        protected override void LifecycleStart(TestLifecycle lifecycle)
+        {
+            base.LifecycleStart(lifecycle);
+            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var contracts = Ci.StartCodexContracts(geth);
+            handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
+        }
+
+        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
+        {
+            base.LifecycleStop(lifecycle, result);
+            handles.Remove(lifecycle);
+        }
+
+        protected IGethNode GetGeth()
+        {
+            return handles[Get()].Geth;
+        }
+
+        protected ICodexContracts GetContracts()
+        {
+            return handles[Get()].Contracts;
+        }
+
+        protected abstract int NumberOfHosts { get; }
+        protected abstract int NumberOfClients { get; }
+        protected abstract ByteSize HostAvailabilitySize { get; }
+        protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
+
+        public ICodexNodeGroup StartHosts()
+        {
+            var hosts = StartCodex(NumberOfHosts, s => s
+                .WithName("host")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsStorageNode()
+                )
+            );
+
+            var config = GetContracts().Deployment.Config;
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei));
+                Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei));
+
+                host.Marketplace.MakeStorageAvailable(new CodexPlugin.StorageAvailability(
+                    totalSpace: HostAvailabilitySize,
+                    maxDuration: HostAvailabilityMaxDuration,
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 999999.Tst())
+                );
+            }
+            return hosts;
+        }
+
+        public TestToken GetTstBalance(ICodexNode node)
+        {
+            return GetContracts().GetTestTokenBalance(node);
+        }
+
+        public TestToken GetTstBalance(EthAddress address)
+        {
+            return GetContracts().GetTestTokenBalance(address);
+        }
+
+        public Ether GetEthBalance(ICodexNode node)
+        {
+            return GetGeth().GetEthBalance(node);
+        }
+
+        public Ether GetEthBalance(EthAddress address)
+        {
+            return GetGeth().GetEthBalance(address);
+        }
+
+        public ICodexNodeGroup StartClients()
+        {
+            return StartCodex(NumberOfClients, s => s
+                .WithName("client")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                )
+            );
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
+        {
+            var fills = GetOnChainSlotFills(possibleHosts);
+            return fills.Where(f => f
+                .SlotFilledEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
+                .ToArray();
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var fills = events.GetSlotFilledEvents();
+            return fills.Select(f =>
+            {
+                var host = possibleHosts.Single(h => h.EthAddress.Address == f.Host.Address);
+                return new SlotFill(f, host);
+
+            }).ToArray();
+        }
+
+        protected void AssertClientHasPaidForContract(TestToken pricePerSlotPerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var balance = GetTstBalance(client);
+            var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerSlotPerSecond, contract, hosts);
+
+            Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect.");
+        }
+
+        protected void AssertHostsWerePaidForContract(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+            var expectedBalances = new Dictionary<EthAddress, TestToken>();
+            foreach (var host in hosts) expectedBalances.Add(host.EthAddress, StartingBalanceTST.Tst());
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            foreach (var pair in expectedBalances)
+            {
+                var balance = GetTstBalance(pair.Key);
+                Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage.");
+            }
+        }
+
+        protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
+        {
+            // There is no separate collateral location yet.
+            // All host balances should be equal to or greater than the starting balance.
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
+            }
+        }
+
+        private TestToken GetContractFinalCost(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var result = 0.Tst();
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                result += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            return result;
+        }
+
+        private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var submitEvent = events.GetStorageRequests().Single(e => e.RequestId.ToHex(false) == contract.PurchaseId);
+            return submitEvent.Block.Utc;
+        }
+
+        private TestToken GetContractCostPerSlot(TestToken pricePerSlotPerSecond, TimeSpan slotDuration)
+        {
+            return pricePerSlotPerSecond * (int)slotDuration.TotalSeconds;
+        }
+
+        protected void AssertContractSlotsAreFilledByHosts(IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var activeHosts = new Dictionary<int, SlotFill>();
+
+            Time.Retry(() =>
+            {
+                var fills = GetOnChainSlotFills(hosts, contract.PurchaseId);
+                foreach (var fill in fills)
+                {
+                    var index = (int)fill.SlotFilledEvent.SlotIndex;
+                    if (!activeHosts.ContainsKey(index))
+                    {
+                        activeHosts.Add(index, fill);
+                    }
+                }
+
+                if (activeHosts.Count != contract.Purchase.MinRequiredNumberOfNodes) throw new Exception("Not all slots were filled...");
+
+            }, nameof(AssertContractSlotsAreFilledByHosts));
+        }
+
+        protected void AssertContractIsOnChain(IStoragePurchaseContract contract)
+        {
+            AssertOnChainEvents(events =>
+            {
+                var onChainRequests = events.GetStorageRequests();
+                if (onChainRequests.Any(r => r.Id == contract.PurchaseId)) return;
+                throw new Exception($"OnChain request {contract.PurchaseId} not found...");
+            }, nameof(AssertContractIsOnChain));
+        }
+
+        protected void AssertOnChainEvents(Action<ICodexContractsEvents> onEvents, string description)
+        {
+            Time.Retry(() =>
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                onEvents(events);
+            }, description);
+        }
+
+        public class SlotFill
+        {
+            public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
+            {
+                SlotFilledEvent = slotFilledEvent;
+                Host = host;
+            }
+
+            public SlotFilledEventDTO SlotFilledEvent { get; }
+            public ICodexNode Host { get; }
+        }
+
+        private class MarketplaceHandle
+        {
+            public MarketplaceHandle(IGethNode geth, ICodexContracts contracts)
+            {
+                Geth = geth;
+                Contracts = contracts;
+            }
+
+            public IGethNode Geth { get; }
+            public ICodexContracts Contracts { get; }
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs

@@ -0,0 +1,83 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class MultipleContractsTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 8;
+        protected override int NumberOfClients => 3;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        [Ignore("TODO - Test where multiple successful contracts are run simultaenously")]
+        public void MultipleSuccessfulContracts()
+        {
+            var hosts = StartHosts();
+            var clients = StartClients();
+
+            var requests = clients.Select(c => CreateStorageRequest(c)).ToArray();
+
+            All(requests, r =>
+            {
+                r.WaitForStorageContractSubmitted();
+                AssertContractIsOnChain(r);
+            });
+
+            All(requests, r => r.WaitForStorageContractStarted());
+            All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts));
+
+            All(requests, r => r.WaitForStorageContractFinished(GetContracts()));
+
+            // todo:
+            //AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            //AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            //AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private void All(IStoragePurchaseContract[] requests, Action<IStoragePurchaseContract> action)
+        {
+            foreach (var r in requests) action(r);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs

@@ -0,0 +1,62 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class BasicInfoTests : CodexDistTest
+    {
+        [Test]
+        public void QuotaTest()
+        {
+            var size = 3.GB();
+            var node = StartCodex(s => s.WithStorageQuota(size));
+            var space = node.Space();
+
+            Assert.That(space.QuotaMaxBytes, Is.EqualTo(size.SizeInBytes));
+        }
+
+        [Test]
+        public void Spr()
+        {
+            var node = StartCodex();
+            
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Spr));
+
+            var spr = node.GetSpr();
+            Assert.That(!string.IsNullOrEmpty(spr));
+
+            Assert.That(info.Spr, Is.EqualTo(spr));
+        }
+
+        [Test]
+        public void VersionInfo()
+        {
+            var node = StartCodex();
+
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Version.Version));
+            Assert.That(!string.IsNullOrEmpty(info.Version.Revision));
+        }
+
+        [Test]
+        public void AnnounceAddress()
+        {
+            var node = StartCodex();
+            var addr = node.Container.GetInternalAddress(CodexContainerRecipe.ListenPortTag);
+
+            var info = node.GetDebugInfo();
+
+            Assert.That(info.AnnounceAddresses.Count, Is.GreaterThan(0));
+            // Ideally we'd assert the pod IP is in the announce address, but we can't access it from here.
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs

@@ -0,0 +1,31 @@
+using CodexPlugin;
+using CodexTests;
+using CodexTests.Helpers;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class PeerTableTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void PeerTableCompleteness()
+        {
+            var nodes = StartCodex(10);
+
+            AssertAllNodesSeeEachOther(nodes.Concat([BootstrapNode!]));
+        }
+
+        private void AssertAllNodesSeeEachOther(IEnumerable<ICodexNode> nodes)
+        {
+            var helper = new PeerConnectionTestHelpers(GetTestLog());
+            helper.AssertFullyConnected(nodes);
+        }
+    }
+}

Tests/CodexReleaseTests/Parallelism.cs

@@ -0,0 +1,6 @@
+using NUnit.Framework;
+
+[assembly: LevelOfParallelism(1)]
+namespace CodexReleaseTests.DataTests
+{
+}

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,54 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class ThreeClientTest : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ThreeClient()
-        {
-            var primary = StartCodex();
-            var secondary = StartCodex();
-
-            var testFile = GenerateTestFile(10.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = secondary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
-
-        [Test]
-        public void DownloadingUnknownCidDoesNotCauseCrash()
-        {
-            var node = StartCodex(2).First();
-
-            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
-
-            try
-            {
-                node.DownloadContent(unknownCid);
-            }
-            catch (Exception ex)
-            {
-                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
-                {
-                    throw;
-                }
-            }
-
-            // Check that the node stays alive for at least another 5 minutes.
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
-            {
-                Thread.Sleep(5000);
-                var info = node.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(info.Id));
-            }
-        }
-    }
-}

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -23,5 +23,21 @@ namespace DistTestCore
             }
             CollectionAssert.IsEmpty(errors);
         }
+
+        public static void AssertLogDoesNotContainLinesStartingWith(this IDownloadedLog log, params string[] unexpectedStrings)
+        {
+            var errors = new List<string>();
+            log.IterateLines(line =>
+            {
+                foreach (var str in unexpectedStrings)
+                {
+                    if (line.StartsWith(str))
+                    {
+                        errors.Add($"Found '{str}' at start of line '{line}'.");
+                    }
+                }
+            });
+            CollectionAssert.IsEmpty(errors);
+        }
     }
 }

Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs

@@ -107,7 +107,7 @@ namespace CodexTests.BasicTests
             AssertStorageRequest(request, purchase, contracts, client);
             AssertContractSlot(contracts, request, 0);
 
-            purchaseContract.WaitForStorageContractFinished();
+            purchaseContract.WaitForStorageContractFinished(contracts);
 
             AssertBalance(contracts, client, Is.LessThan(clientInitialBalance), "Buyer was not charged for storage.");
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));

Tests/ExperimentalTests/CodexDistTest.cs

@@ -99,7 +99,7 @@ namespace CodexTests
             var log = Ci.DownloadLog(node);
 
             log.AssertLogDoesNotContain("Block validation failed");
-            log.AssertLogDoesNotContain("ERR ");
+            log.AssertLogDoesNotContainLinesStartingWith("ERR ");
         }
 
         public void LogNodeStatus(ICodexNode node, IMetricsAccess? metrics = null)
@@ -108,6 +108,39 @@ namespace CodexTests
                 GetBasicNodeStatus(node));
         }
 
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes)
+        {
+            WaitAndCheckNodesStaysAlive(duration, nodes.ToArray());
+        }
+
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes)
+        {
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < duration)
+            {
+                Thread.Sleep(5000);
+                foreach (var node in nodes)
+                {
+                    var info = node.GetDebugInfo();
+                    Assert.That(!string.IsNullOrEmpty(info.Id));
+                }
+            }
+        }
+
+        public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes)
+        {
+            AssertNodesContainFile(cid, nodes.ToArray());
+        }
+
+        public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                var localDatasets = node.LocalFiles();
+                CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
+            }
+        }
+
         private string GetBasicNodeStatus(ICodexNode node)
         {
             return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine +

Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs

@@ -4,7 +4,7 @@ using Utils;
 namespace CodexTests.DownloadConnectivityTests
 {
     [TestFixture]
-    public class SwarmTests : AutoBootstrapDistTest
+    public class DetectBlockRetransmitTest : AutoBootstrapDistTest
     {
         [Test]
         [Combinatorial]

Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs

@@ -45,7 +45,7 @@ namespace CodexTests.UtilityTests
 
             var purchaseContract = ClientPurchasesStorage(client);
             purchaseContract.WaitForStorageContractStarted();
-            purchaseContract.WaitForStorageContractFinished();
+            purchaseContract.WaitForStorageContractFinished(contracts);
             Thread.Sleep(rewarderInterval * 3);
 
             apiCalls.Stop();

Tools/BiblioTech/CodexCidChecker.cs

@@ -10,6 +10,7 @@ namespace BiblioTech
         private static readonly string nl = Environment.NewLine;
         private readonly Configuration config;
         private readonly ILog log;
+        private readonly Mutex checkMutex = new Mutex();
         private CodexApi? currentCodexNode;
 
         public CodexCidChecker(Configuration config, ILog log)
@@ -27,6 +28,7 @@ namespace BiblioTech
 
             try
             {
+                checkMutex.WaitOne();
                 var codex = GetCodex();
                 var nodeCheck = await CheckCodex(codex);
                 if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
@@ -37,6 +39,10 @@ namespace BiblioTech
             {
                 return new CheckResponse(false, "Internal server error", ex.ToString());
             }
+            finally
+            {
+                checkMutex.ReleaseMutex();
+            }
         }
 
         private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -1,4 +1,8 @@
 using BiblioTech.Options;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Discord;
 
 namespace BiblioTech.Commands
 {
@@ -9,10 +13,12 @@ namespace BiblioTech.Commands
             description: "Codex Content-Identifier",
             isRequired: true);
         private readonly CodexCidChecker checker;
+        private readonly CidStorage cidStorage;
 
         public CheckCidCommand(CodexCidChecker checker)
         {
             this.checker = checker;
+            this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt"));
         }
 
         public override string Name => "check";
@@ -32,7 +38,85 @@ namespace BiblioTech.Commands
 
             var response = await checker.PerformCheck(cid);
             await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
+
+            if (response.Success)
+            {
+                await CheckAltruisticRole(context, user, cid, response.Message);
+                return;
+            }
+
             await context.Followup(response.Message);
         }
+
+        private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage)
+        {
+            if (cidStorage.TryAddCid(cid, user.Id))
+            {
+                if (await GiveAltruisticRole(context, user, responseMessage))
+                {
+                    return;
+                }
+            }
+            else
+            {
+                await context.Followup($"{responseMessage}\n\nThis CID has already been used by another user. No role will be granted.");
+                return;
+            }
+
+            await context.Followup(responseMessage);
+        }
+
+        private async Task<bool> GiveAltruisticRole(CommandContext context, IUser user, string responseMessage)
+        {
+            var guildUser = context.Command.User as IGuildUser;
+            if (guildUser != null)
+            {
+                try
+                {
+                    var role = context.Command.Guild.GetRole(Program.Config.AltruisticRoleId);
+                    if (role != null)
+                    {
+                        await guildUser.AddRoleAsync(role);
+                        await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!");
+                        return true;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}");
+                }
+            }
+            return false;
+        }
+    }
+
+    public class CidStorage
+    {
+        private readonly string filePath;
+        private static readonly object _lock = new object();
+
+        public CidStorage(string filePath)
+        {
+            this.filePath = filePath;
+            if (!File.Exists(filePath))
+            {
+                File.WriteAllText(filePath, string.Empty);
+            }
+        }
+
+        public bool TryAddCid(string cid, ulong userId)
+        {
+            lock (_lock)
+            {
+                var existingEntries = File.ReadAllLines(filePath);
+                if (existingEntries.Any(line => line.Split(',')[0] == cid))
+                {
+                    return false;
+                }
+
+                File.AppendAllLines(filePath, new[] { $"{cid},{userId}" });
+                return true;
+            }
+        }
     }
 }

Tools/BiblioTech/Configuration.cs

@@ -26,6 +26,9 @@ namespace BiblioTech
         [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")]
         public ulong ChainEventsChannelId { get; set; }
 
+        [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")]
+        public ulong AltruisticRoleId { get; set; }
+
         [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")]
         public int RewardApiPort { get; set; } = 31080;
 

Tools/CodexNetDeployer/CodexNetDeployer.csproj

@@ -12,7 +12,7 @@
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\..\Tests\CodexTests\CodexTests.csproj" />
+    <ProjectReference Include="..\..\Tests\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tools/TestNetRewarder/Configuration.cs

@@ -4,6 +4,8 @@ namespace TestNetRewarder
 {
     public class Configuration
     {
+        private readonly DateTime AppStartUct = DateTime.UtcNow;
+
         [Uniform("datapath", "dp", "DATAPATH", true, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
 
@@ -16,8 +18,8 @@ namespace TestNetRewarder
         [Uniform("interval-minutes", "im", "INTERVALMINUTES", true, "time in minutes between reward updates.")]
         public int IntervalMinutes { get; set; } = 15;
 
-        [Uniform("check-history", "ch", "CHECKHISTORY", true, "Unix epoc timestamp of a moment in history on which processing begins. Required for hosting rewards. Should be 'launch of the testnet'.")]
-        public int CheckHistoryTimestamp { get; set; } = 0;
+        [Uniform("relative-history", "rh", "RELATIVEHISTORY", false, "Number of seconds into the past (from app start) that checking of chain history will start. Default: 3 hours ago.")]
+        public int RelativeHistorySeconds { get; set; } = 3600 * 3;
 
         [Uniform("market-insights", "mi", "MARKETINSIGHTS", false, "Semi-colon separated integers. Each represents a multiple of intervals, for which a market insights average will be generated.")]
         public string MarketInsights { get; set; } = "1;96";
@@ -45,8 +47,7 @@ namespace TestNetRewarder
         {
             get
             {
-                if (CheckHistoryTimestamp == 0) throw new Exception("'check-history' unix timestamp is required. Set it to the start/launch moment of the testnet.");
-                return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime;
+                return AppStartUct - TimeSpan.FromSeconds(RelativeHistorySeconds);
             }
         }
     }

Tools/TestNetRewarder/EmojiMaps.cs

@@ -1,22 +1,11 @@
-using Utils;
-
-namespace TestNetRewarder
+namespace TestNetRewarder
 {
     public class EmojiMaps
     {
         private readonly string[] emojis = new[]
         {
-            // yellow
-            "😀",
-            "🌻",
-            "🍋",
-            "🧀",
-            "🌔",
-            "⭐",
-            "⚡",
-            "🏆",
             // red
-            "💘",
+            "❤",
             "🦞",
             "🌹",
             "🍒",
@@ -24,8 +13,29 @@ namespace TestNetRewarder
             "⛩",
             "🚗",
             "🔥",
+
+            // orange
+            "🧡",
+            "🏀",
+            "🦊",
+            "🏵",
+            "🍊",
+            "🥕",
+            "🧱",
+            "🎃",
+
+            // yellow
+            "💛",
+            "🌻",
+            "🍋",
+            "🧀",
+            "🌔",
+            "⭐",
+            "⚡",
+            "🏆",
+
             // green
-            "🐊",
+            "💚",
             "🦎",
             "🐛",
             "🌳",
@@ -33,19 +43,40 @@ namespace TestNetRewarder
             "🧩",
             "🔋",
             "♻",
+
             // blue
             "💙",
             "🐳",
-            "🐟",
+            "♂",
             "🍉",
             "🧊",
             "🌐",
             "⚓",
             "🌀",
+
+            // purple
+            "💜",
+            "🪀", //yo-yo
+            "🔮",
+            "😈",
+            "👾",
+            "🪻", // plant hyacinth
+            "🍇",
+            "🍆",
+
+            // pink
+            "🩷", // pink heart
+            "👚",
+            "♀",
+            "🧠",
+            "🐷",
+            "🦩",
+            "🌸",
+            "🌷"
         };
 
-        public string NewRequest => "🐟";
-        public string Started => "🦈";
+        public string NewRequest => "🌱";
+        public string Started => "🌳";
         public string SlotFilled => "🟢";
         public string SlotFreed => "⭕";
         public string SlotReservationsFull => "☑️";

cs-codex-dist-testing.sln

@@ -37,8 +37,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexContinuousTests", "Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexTestsLong", "Tests\CodexLongTests\CodexTestsLong.csproj", "{0C2D067F-053C-45A8-AE0D-4EB388E77C89}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexTests", "Tests\CodexTests\CodexTests.csproj", "{562EC700-6984-4C9A-83BF-3BF4E3EB1A64}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "Tests\DistTestCore\DistTestCore.csproj", "{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "Tools\CodexNetDeployer\CodexNetDeployer.csproj", "{3417D508-E2F4-4974-8988-BB124046D9E2}"
@@ -76,7 +74,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexReleaseTests", "Tests\CodexReleaseTests\CodexReleaseTests.csproj", "{639A0603-4E80-465B-BB59-AB02F1DEEF5A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExperimentalTests", "Tests\ExperimentalTests\ExperimentalTests.csproj", "{BA7369CD-7C2F-4075-8E35-98BCC19EE203}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -136,10 +138,6 @@ Global
 		{0C2D067F-053C-45A8-AE0D-4EB388E77C89}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0C2D067F-053C-45A8-AE0D-4EB388E77C89}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0C2D067F-053C-45A8-AE0D-4EB388E77C89}.Release|Any CPU.Build.0 = Release|Any CPU
-		{562EC700-6984-4C9A-83BF-3BF4E3EB1A64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{562EC700-6984-4C9A-83BF-3BF4E3EB1A64}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{562EC700-6984-4C9A-83BF-3BF4E3EB1A64}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{562EC700-6984-4C9A-83BF-3BF4E3EB1A64}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -208,6 +206,14 @@ Global
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
+		{639A0603-4E80-465B-BB59-AB02F1DEEF5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{639A0603-4E80-465B-BB59-AB02F1DEEF5A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{639A0603-4E80-465B-BB59-AB02F1DEEF5A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{639A0603-4E80-465B-BB59-AB02F1DEEF5A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -226,7 +232,6 @@ Global
 		{8DE8FF65-23CB-4FB3-8BE5-6C0BEC4BAA97} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
 		{ADEC06CF-6F3A-44C5-AA57-EAB94124AC82} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 		{0C2D067F-053C-45A8-AE0D-4EB388E77C89} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
-		{562EC700-6984-4C9A-83BF-3BF4E3EB1A64} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 		{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 		{3417D508-E2F4-4974-8988-BB124046D9E2} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{078ABA6D-A04E-4F62-A44C-EA66F1B66548} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
@@ -244,6 +249,8 @@ Global
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{639A0603-4E80-465B-BB59-AB02F1DEEF5A} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
+		{BA7369CD-7C2F-4075-8E35-98BCC19EE203} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}

docker/docker-entrypoint.sh

@@ -1,35 +1,33 @@
 #!/bin/bash
 
-# Common
+# Variables
+## Common
 SOURCE="${SOURCE:-https://github.com/codex-storage/cs-codex-dist-tests.git}"
 BRANCH="${BRANCH:-master}"
 FOLDER="${FOLDER:-/opt/cs-codex-dist-tests}"
 
-# Continuous Tests
+## Tests specific
 DEPLOYMENT_CODEXNETDEPLOYER_PATH="${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}"
 DEPLOYMENT_CODEXNETDEPLOYER_RUNNER="${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER:-deploy-continuous-testnet.sh}"
 CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/CodexContinuousTests}"
 CONTINUOUS_TESTS_RUNNER="${CONTINUOUS_TESTS_RUNNER:-run.sh}"
 
-
 # Get code
-echo "`date` - Clone ${SOURCE}"
+echo -e "Cloning ${SOURCE} to ${FOLDER}\n"
 git clone -b "${BRANCH}" "${SOURCE}" "${FOLDER}"
-echo "`date` - Change folder to ${FOLDER}"
+echo -e "\nChanging folder to ${FOLDER}\n"
 cd "${FOLDER}"
 
-# Run
-echo "Run tests from branch '`git branch --show-current` / `git rev-parse HEAD`'"
+# Run tests
+echo -e "Running tests from branch '$(git branch --show-current) ($(git rev-parse --short HEAD))'\n"
 
 if [[ "${TESTS_TYPE}" == "continuous-tests" ]]; then
-  echo "`date` - Running Continuous Tests"
-  echo
-  echo "`date` - Running CodexNetDeployer"
+  echo -e "Running CodexNetDeployer\n"
   bash "${DEPLOYMENT_CODEXNETDEPLOYER_PATH}"/"${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER}"
   echo
-  echo "`date` - Running Tests"
+  echo -e "Running continuous-tests\n"
   bash "${CONTINUOUS_TESTS_FOLDER}"/"${CONTINUOUS_TESTS_RUNNER}"
 else
-  echo "`date` - Running Dist Tests"
+  echo -e "Running ${TESTS_TYPE}\n"
   exec "$@"
 fi

docker/job-release-tests.yaml

@@ -0,0 +1,63 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAMEPREFIX}
+  namespace: ${NAMESPACE}
+  labels:
+    name: ${NAMEPREFIX}
+    runid: ${RUNID}
+spec:
+  ttlSecondsAfterFinished: 86400
+  backoffLimit: 0
+  template:
+    metadata:
+      name: ${NAMEPREFIX}
+      labels:
+        app: ${TEST_TYPE}-runner
+        name: ${NAMEPREFIX}
+        runid: ${RUNID}
+    spec:
+      priorityClassName: system-node-critical
+      nodeSelector:
+        workload-type: "tests-runners-ci"
+      containers:
+      - name: runner
+        image: codexstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "1Gi"
+        env:
+        - name: KUBECONFIG
+          value: "/opt/kubeconfig.yaml"
+        - name: LOGPATH
+          value: "/var/log/codex-${TEST_TYPE}"
+        - name: NAMESPACE
+          value: "${NAMESPACE}"
+        - name: BRANCH
+          value: "${BRANCH}"
+        - name: SOURCE
+          value: "${SOURCE}"
+        - name: RUNID
+          value: "${RUNID}"
+        - name: CODEXDOCKERIMAGE
+          value: "${CODEXDOCKERIMAGE}"
+        - name: TESTID
+          value: "${TESTID}"
+        - name: TESTS_TYPE
+          value: "${TEST_TYPE}"
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/codex-${TEST_TYPE}
+        args: ${COMMAND}
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/codex-${TEST_TYPE}

docs/CodexHowTo.md

@@ -0,0 +1,9 @@
+# Codex Tests - How To
+Follow these steps to run the distributed tests for Codex. These tests should pass for any (future) release of Codex.
+
+## Build a docker image.
+If you already have a dist-tests compatible docker image, skip this step.
+![BuildImage](./CodexHowTo_BuildImage.png)
+
+## Run tests.
+![RunTests](./CodexHowTo_RunReleaseTests.png)