Merge branch 'master' into feature/proofs-and-frees

# Conflicts:
#	Framework/KubernetesWorkflow/LogHandler.cs
#	Framework/Utils/EthAddress.cs
#	ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs
#	Tests/CodexReleaseTests/MarketTests/FailTest.cs
#	Tests/CodexReleaseTests/MarketTests/FinishTest.cs
#	Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs
#	Tools/AutoClient/Modes/FolderStore/FileSaver.cs

.github/workflows/trace-contract.yaml

@@ -0,0 +1,43 @@
+name: Trace Contract
+
+on:
+  workflow_dispatch:
+    inputs:
+      purchaseid:
+        description: "Testnet Purchase ID"
+        required: true
+        type: string
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  OUTPUT_FOLDER: output
+  ES_USERNAME: ${{ secrets.ES_USERNAME }}
+  ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
+  ES_HOST: ${{ secrets.ES_HOST }}
+
+jobs:
+  trace_contract:
+    name: Trace contract
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          echo "PURCHASE_ID=${{ inputs.purchaseid }}" >> $GITHUB_ENV
+
+      - name: Run Trace
+        run: |
+          dotnet run --project Tools/TraceContract
+
+      - name: Upload output
+        uses: actions/upload-artifact@v4
+        with:
+          name: contract-trace
+          path: ${{ env.OUTPUT_FOLDER }}/
+          if-no-files-found: error
+          retention-days: 7

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -20,9 +20,10 @@ namespace BlockchainUtils
 
         public BlockTimeEntry Get(ulong blockNumber)
         {
-            bounds.Initialize();
             var b = cache.Get(blockNumber);
             if (b != null) return b;
+
+            bounds.Initialize();
             return GetBlock(blockNumber);
         }
 
@@ -101,10 +102,18 @@ namespace BlockchainUtils
                 previous.Utc < target;
         }
 
-        private BlockTimeEntry GetBlock(ulong number)
+        private BlockTimeEntry GetBlock(ulong number, bool retry = false)
         {
             if (number < bounds.Genesis.BlockNumber) throw new Exception("Can't fetch block before genesis.");
-            if (number > bounds.Current.BlockNumber) throw new Exception("Can't fetch block after current.");
+            if (number > bounds.Current.BlockNumber)
+            {
+                if (retry) throw new Exception("Can't fetch block after current.");
+
+                // todo test and verify this:
+                Thread.Sleep(1000);
+                bounds.Initialize();
+                return GetBlock(number, retry: true);
+            }
 
             var dateTime = web3.GetTimestampForBlock(number);
             if (dateTime == null) throw new Exception("Failed to get dateTime for block that should exist.");

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -87,6 +87,8 @@
         private void AddCurrentBlock()
         {
             var currentBlockNumber = web3.GetCurrentBlockNumber();
+            if (Current != null && Current.BlockNumber == currentBlockNumber) return;
+
             var blockTime = web3.GetTimestampForBlock(currentBlockNumber);
             if (blockTime == null) throw new Exception("Unable to get dateTime for current block.");
             AddCurrentBlock(currentBlockNumber, blockTime.Value);

Framework/KubernetesWorkflow/ContainerCrashWatcher.cs

@@ -14,6 +14,7 @@ namespace KubernetesWorkflow
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
+        private bool hasCrashed = false;
 
         public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
@@ -47,10 +48,7 @@ namespace KubernetesWorkflow
 
         public bool HasCrashed()
         {
-            using var client = new Kubernetes(config);
-            var result = HasContainerBeenRestarted(client);
-            if (result) DownloadCrashedContainerLogs(client);
-            return result;
+            return hasCrashed;
         }
 
         private void Worker()
@@ -72,6 +70,9 @@ namespace KubernetesWorkflow
             {
                 if (HasContainerBeenRestarted(client))
                 {
+                    hasCrashed = true;
+                    cts.Cancel();
+
                     DownloadCrashedContainerLogs(client);
                     return;
                 }

Framework/KubernetesWorkflow/LogHandler.cs

@@ -33,7 +33,7 @@ namespace KubernetesWorkflow
             sourceLog.Log(msg);
 
             LogFile.Write(msg);
-            LogFile.WriteRaw(description);
+            LogFile.Write(description);
         }
 
         public LogFile LogFile { get; }
@@ -43,8 +43,9 @@ namespace KubernetesWorkflow
             // This line is not useful and has no topic so we can't filter it with
             // normal log-level controls.
             if (line.Contains("Received JSON-RPC response") && !line.Contains("topics=")) return;
+            if (line.Contains("object field not marked with serialize, skipping")) return;
 
-            LogFile.WriteRaw(line);
+            LogFile.Write(line);
         }
     }
 }

Framework/Logging/BaseLog.cs

@@ -65,7 +65,7 @@ namespace Logging
 
         public void Raw(string message)
         {
-            LogFile.WriteRaw(message);
+            LogFile.Write(message);
         }
 
         public virtual void AddStringReplace(string from, string to)

Framework/Logging/LogFile.cs

@@ -1,6 +1,4 @@
-using Utils;
-
-namespace Logging
+namespace Logging
 {
     public class LogFile
     {
@@ -16,11 +14,6 @@ namespace Logging
         public string Filename { get; private set; }
 
         public void Write(string message)
-        {
-            WriteRaw($"{GetTimestamp()} {message}");
-        }
-
-        public void WriteRaw(string message)
         {
             try
             {
@@ -50,11 +43,6 @@ namespace Logging
             }
         }
 
-        private static string GetTimestamp()
-        {
-            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
-        }
-
         private void EnsurePathExists(string filename)
         {
             var path = new FileInfo(filename).Directory!.FullName;

Framework/Logging/LogPrefixer.cs

@@ -24,17 +24,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(Prefix + message, skipFrames);
+            backingLog.Debug(GetPrefix() + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(Prefix + message);
+            backingLog.Error(GetPrefix() + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(Prefix + message);
+            backingLog.Log(GetPrefix() + message);
         }
 
         public void AddStringReplace(string from, string to)
@@ -51,5 +51,10 @@
         {
             return backingLog.GetFullName();
         }
+
+        protected virtual string GetPrefix()
+        {
+            return Prefix;
+        }
     }
 }

Framework/Logging/TimestampPrefixer.cs

@@ -0,0 +1,16 @@
+using Utils;
+
+namespace Logging
+{
+    public class TimestampPrefixer : LogPrefixer
+    {
+        public TimestampPrefixer(ILog backingLog) : base(backingLog)
+        {
+        }
+
+        protected override string GetPrefix()
+        {
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -143,5 +143,16 @@ namespace NethereumWorkflow
             var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
             return blockTimeFinder.Get(number);
         }
+
+        public BlockWithTransactions GetBlockWithTransactions(ulong number)
+        {
+            var retry = new Retry(nameof(GetBlockWithTransactions),
+                maxTimeout: TimeSpan.FromMinutes(1.0),
+                sleepAfterFail: TimeSpan.FromSeconds(1.0),
+                onFail: f => { },
+                failFast: false);
+
+            return retry.Run(() => Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(number))));
+        }
     }
 }

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -19,23 +19,40 @@ namespace NethereumWorkflow
 
         public ulong GetCurrentBlockNumber()
         {
-            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
-            return Convert.ToUInt64(number.ToDecimal());
+            return Retry(() =>
+            {
+                var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+                return Convert.ToUInt64(number.ToDecimal());
+            });
         }
 
         public DateTime? GetTimestampForBlock(ulong blockNumber)
         {
-            try
+            return Retry<DateTime?>(() =>
             {
-                var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
-                if (block == null) return null;
-                return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
-            }
-            catch (Exception ex)
-            {
-                log.Error("Exception while getting timestamp for block: " + ex);
-                return null;
-            }
+                try
+                {
+                    var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
+                    if (block == null) return null;
+                    return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Exception while getting timestamp for block: " + ex);
+                    return null;
+                }
+            });
+        }
+
+        private T Retry<T>(Func<T> action)
+        {
+            var retry = new Retry(nameof(Web3Wrapper),
+                maxTimeout: TimeSpan.FromSeconds(30),
+                sleepAfterFail: TimeSpan.FromSeconds(3),
+                onFail: f => { },
+                failFast: false);
+
+            return retry.Run(action);
         }
     }
 }

Framework/Utils/ByteSize.cs

@@ -25,6 +25,21 @@
             return new ByteSize(Convert.ToInt64(result));
         }
 
+        public int DivUp(ByteSize div)
+        {
+            var d = div.SizeInBytes;
+            var remaining = SizeInBytes;
+            var result = 0;
+            while (remaining > d)
+            {
+                remaining -= d;
+                result++;
+            }
+
+            if (remaining > 0) result++;
+            return result;
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is ByteSize size && SizeInBytes == size.SizeInBytes;

Framework/Utils/EthAccount.cs

@@ -1,7 +1,7 @@
 namespace Utils
 {
     [Serializable]
-    public class EthAccount
+    public class EthAccount : IComparable<EthAccount>
     {
         public EthAccount(EthAddress ethAddress, string privateKey)
         {
@@ -12,9 +12,37 @@
         public EthAddress EthAddress { get; }
         public string PrivateKey { get; }
 
+        public int CompareTo(EthAccount? other)
+        {
+            return PrivateKey.CompareTo(other!.PrivateKey);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is EthAccount token && PrivateKey == token.PrivateKey;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(PrivateKey);
+        }
+
         public override string ToString()
         {
             return EthAddress.ToString();
         }
+
+        public static bool operator ==(EthAccount? a, EthAccount? b)
+        {
+            if (ReferenceEquals(a, b)) return true;
+            if (ReferenceEquals(a, null)) return false;
+            if (ReferenceEquals(b, null)) return false;
+            return a.PrivateKey == b.PrivateKey;
+        }
+
+        public static bool operator !=(EthAccount? a, EthAccount? b)
+        {
+            return !(a == b);
+        }
     }
 }

Framework/Utils/EthAddress.cs

@@ -6,7 +6,7 @@
     }
 
     [Serializable]
-    public class EthAddress
+    public class EthAddress : IComparable<EthAddress>
     {
         public EthAddress(string address)
         {
@@ -15,10 +15,14 @@
 
         public string Address { get; }
 
+        public int CompareTo(EthAddress? other)
+        {
+            return Address.CompareTo(other!.Address);
+        }
+
         public override bool Equals(object? obj)
         {
-            return obj is EthAddress address &&
-                   Address == address.Address;
+            return obj is EthAddress token && Address == token.Address;
         }
 
         public override int GetHashCode()
@@ -31,14 +35,17 @@
             return Address;
         }
 
-        public static bool operator ==(EthAddress a, EthAddress b)
+        public static bool operator ==(EthAddress? a, EthAddress? b)
         {
+            if (ReferenceEquals(a, b)) return true;
+            if (ReferenceEquals(a, null)) return false;
+            if (ReferenceEquals(b, null)) return false;
             return a.Address == b.Address;
         }
 
-        public static bool operator !=(EthAddress a, EthAddress b)
+        public static bool operator !=(EthAddress? a, EthAddress? b)
         {
-            return a.Address != b.Address;
+            return !(a == b);
         }
     }
 }

Framework/Utils/Str.cs

@@ -4,8 +4,11 @@
     {
         public static string Between(string input, string open, string close)
         {
-            var openIndex = input.IndexOf(open) + open.Length;
+            var openI = input.IndexOf(open);
+            if (openI == -1) return input;
+            var openIndex = openI + open.Length;
             var closeIndex = input.LastIndexOf(close);
+            if (closeIndex == -1) return input;
 
             return input.Substring(openIndex, closeIndex - openIndex);
         }

Framework/WebUtils/Endpoint.cs

@@ -85,7 +85,7 @@ namespace WebUtils
                 var result = Deserialize<TResponse>(response);
                 if (result == null) throw new Exception("Failed to deserialize response");
                 return result;
-            }, $"HTTO-POST-JSON: {route}");
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)

ProjectPlugins/CodexClient/CodexNode.cs

@@ -318,6 +318,14 @@ namespace CodexClient
             log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
             log.AddStringReplace(nodeId, nodeName);
             log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName);
+
+            var ethAccount = codexAccess.GetEthAccount();
+            if (ethAccount != null)
+            {
+                var addr = ethAccount.EthAddress.ToString();
+                log.AddStringReplace(addr, nodeName);
+                log.AddStringReplace(addr.ToLowerInvariant(), nodeName);
+            }
         }
 
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)

ProjectPlugins/CodexClient/CodexTypes.cs

@@ -17,6 +17,7 @@ namespace CodexClient
     {
         public string Version { get; set; } = string.Empty;
         public string Revision { get; set; } = string.Empty;
+        public string Contracts { get; set; } = string.Empty;
 
         public bool IsValid()
         {

ProjectPlugins/CodexClient/Mapper.cs

@@ -166,7 +166,8 @@ namespace CodexClient
             return new DebugInfoVersion
             {
                 Version = obj.Version,
-                Revision = obj.Revision
+                Revision = obj.Revision,
+                Contracts = obj.Contracts
             };
         }
 

ProjectPlugins/CodexClient/openapi.yaml

@@ -124,6 +124,9 @@ components:
         revision:
           type: string
           example: 0c647d8
+        contracts:
+          type: string
+          example: 0b537c7
 
     PeersTable:
       type: object
@@ -202,6 +205,7 @@ components:
       required:
         - id
         - totalRemainingCollateral
+        - freeSize
       allOf:
         - $ref: "#/components/schemas/SalesAvailability"
         - type: object
@@ -624,6 +628,26 @@ paths:
         "500":
           description: Well it was bad-bad
 
+    delete:
+      summary: "Deletes either a single block or an entire dataset from the local node."
+      tags: [Data]
+      operationId: deleteLocal
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: Block or dataset to be deleted.
+
+      responses:
+        "204":
+          description: Data was successfully deleted.
+        "400":
+          description: Invalid CID is specified
+        "500":
+          description: There was an error during deletion
+
   "/data/{cid}/network":
     post:
       summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -47,7 +47,7 @@ namespace CodexContractsPlugin.ChainMonitor
             handler = changeHandler;
             this.doProofPeriodMonitoring = doProofPeriodMonitoring;
             TotalSpan = new TimeRange(startUtc, startUtc);
-            PeriodMonitor = new PeriodMonitor(this.log, contracts);
+            PeriodMonitor = new PeriodMonitor(contracts);
         }
 
         public TimeRange TotalSpan { get; private set; }
@@ -199,21 +199,23 @@ namespace CodexContractsPlugin.ChainMonitor
         private ChainStateRequest? FindRequest(IHasRequestId request)
         {
             var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
-            if (r == null)
+            if (r != null) return r;
+           
+            try
             {
-                var blockNumber = "unknown";
-                if (request is IHasBlock blk)
-                {
-                    blockNumber = blk.Block.BlockNumber.ToString();
-                }
-
-                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
-                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
-
+                var req = contracts.GetRequest(request.RequestId);
+                var state = contracts.GetRequestState(req);
+                var newRequest = new ChainStateRequest(log, req, state);
+                requests.Add(newRequest);
+                return newRequest;
+            }
+            catch (Exception ex)
+            {
+                var msg = "Failed to get request from chain: " + ex;
                 log.Error(msg);
                 handler.OnError(msg);
+                return null;
             }
-            return r;
         }
 
         private bool Equal(byte[] a, byte[] b)

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs

@@ -1,18 +1,15 @@
-using Logging;
-using Utils;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
     public class PeriodMonitor
     {
-        private readonly ILog log;
         private readonly ICodexContracts contracts;
         private readonly List<PeriodReport> reports = new List<PeriodReport>();
         private ulong? currentPeriod = null;
 
-        public PeriodMonitor(ILog log, ICodexContracts contracts)
+        public PeriodMonitor(ICodexContracts contracts)
         {
-            this.log = log;
             this.contracts = contracts;
         }
 
@@ -87,6 +84,8 @@ namespace CodexContractsPlugin.ChainMonitor
         private void CalcStats()
         {
             IsEmpty = Reports.All(r => r.TotalProofsRequired == 0);
+            if (Reports.Length == 0) return;
+
             PeriodLow = Reports.Min(r => r.PeriodNumber);
             PeriodHigh = Reports.Max(r => r.PeriodNumber);
             AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots));

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -25,6 +25,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        Request GetRequest(byte[] requestId);
         ulong GetPeriodNumber(DateTime utc);
         void WaitUntilNextPeriod();
         ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
@@ -123,6 +124,17 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public Request GetRequest(byte[] requestId)
+        {
+            var func = new GetRequestFunction
+            {
+                RequestId = requestId
+            };
+
+            var request = gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(Deployment.MarketplaceAddress, func);
+            return request.ReturnValue1;
+        }
+
         public ulong GetPeriodNumber(DateTime utc)
         {
             DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,4 +1,5 @@
-using GethPlugin;
+using CodexClient;
+using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Recipe;
 
@@ -8,14 +9,14 @@ namespace CodexContractsPlugin
     {
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
-        private readonly VersionRegistry versionRegistry;
+        private readonly DebugInfoVersion versionInfo;
 
         public override string AppName => "codex-contracts";
-        public override string Image => versionRegistry.GetContractsDockerImage();
+        public override string Image => GetContractsDockerImage();
 
-        public CodexContractsContainerRecipe(VersionRegistry versionRegistry)
+        public CodexContractsContainerRecipe(DebugInfoVersion versionInfo)
         {
-            this.versionRegistry = versionRegistry;
+            this.versionInfo = versionInfo;
         }
 
         protected override void Initialize(StartupConfig startupConfig)
@@ -28,7 +29,13 @@ namespace CodexContractsPlugin
 
             AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
+            AddEnvVar("HARDHAT_IGNITION_CONFIRM_DEPLOYMENT", "false");
             AddEnvVar("KEEP_ALIVE", "1");
         }
+
+        private string GetContractsDockerImage()
+        {
+            return $"codexstorage/codex-contracts-eth:sha-{versionInfo.Contracts}-dist-tests";
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -19,6 +19,7 @@ namespace CodexContractsPlugin
         SlotFreedEventDTO[] GetSlotFreedEvents();
         SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents();
         ProofSubmittedEventDTO[] GetProofSubmittedEvents();
+        void GetReserveSlotCalls(Action<ReserveSlotFunction> onFunction);
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -99,6 +100,15 @@ namespace CodexContractsPlugin
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
+        public void GetReserveSlotCalls(Action<ReserveSlotFunction> onFunction)
+        {
+            gethNode.IterateFunctionCalls<ReserveSlotFunction>(BlockInterval, (b, fn) =>
+            {
+                fn.Block = b;
+                onFunction(fn);
+            });
+        }
+
         private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
         {
             var result = e.Event;

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs

@@ -7,15 +7,11 @@ namespace CodexContractsPlugin
     {
         private readonly IPluginTools tools;
         private readonly CodexContractsStarter starter;
-        private readonly VersionRegistry versionRegistry;
-        private readonly CodexContractsContainerRecipe recipe;
 
         public CodexContractsPlugin(IPluginTools tools)
         {
             this.tools = tools;
-            versionRegistry = new VersionRegistry(tools.GetLog());
-            recipe = new CodexContractsContainerRecipe(versionRegistry);
-            starter = new CodexContractsStarter(tools, recipe);
+            starter = new CodexContractsStarter(tools);
         }
 
         public string LogPrefix => "(CodexContracts) ";
@@ -31,16 +27,16 @@ namespace CodexContractsPlugin
 
         public void AddMetadata(IAddMetadata metadata)
         {
-            metadata.Add("codexcontractsid", recipe.Image);
+            metadata.Add("codexcontractsid", "dynamic");
         }
 
         public void Decommission()
         {
         }
 
-        public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode)
+        public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode, CodexClient.DebugInfoVersion versionInfo)
         {
-            return starter.Deploy(ci, gethNode);
+            return starter.Deploy(ci, gethNode, versionInfo);
         }
 
         public ICodexContracts WrapDeploy(IGethNode gethNode, CodexContractsDeployment deployment)
@@ -48,10 +44,5 @@ namespace CodexContractsPlugin
             deployment = SerializeGate.Gate(deployment);
             return starter.Wrap(gethNode, deployment);
         }
-
-        public void SetCodexDockerImageProvider(ICodexDockerImageProvider provider)
-        {
-            versionRegistry.SetProvider(provider);
-        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+    <ProjectReference Include="..\CodexClient\CodexClient.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
   </ItemGroup>
 

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using CodexClient;
+using CodexContractsPlugin.Marketplace;
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
@@ -12,15 +13,13 @@ namespace CodexContractsPlugin
     public class CodexContractsStarter
     {
         private readonly IPluginTools tools;
-        private readonly CodexContractsContainerRecipe recipe;
 
-        public CodexContractsStarter(IPluginTools tools, CodexContractsContainerRecipe recipe)
+        public CodexContractsStarter(IPluginTools tools)
         {
             this.tools = tools;
-            this.recipe = recipe;
         }
 
-        public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode)
+        public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
         {
             Log("Starting Codex SmartContracts container...");
 
@@ -28,6 +27,9 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
+            var recipe = new CodexContractsContainerRecipe(versionInfo);
+            Log($"Using image: {recipe.Image}");
+
             var containers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];

ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs

@@ -1,13 +1,14 @@
-using Core;
+using CodexClient;
+using Core;
 using GethPlugin;
 
 namespace CodexContractsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode)
+        public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
         {
-            return Plugin(ci).DeployContracts(ci, gethNode);
+            return Plugin(ci).DeployContracts(ci, gethNode, versionInfo);
         }
 
         public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, IGethNode gethNode, CodexContractsDeployment deployment)
@@ -15,17 +16,12 @@ namespace CodexContractsPlugin
             return Plugin(ci).WrapDeploy(gethNode, deployment);
         }
 
-        public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode)
+        public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo)
         {
-            var deployment = DeployCodexContracts(ci, gethNode);
+            var deployment = DeployCodexContracts(ci, gethNode, versionInfo);
             return WrapCodexContractsDeployment(ci, gethNode, deployment);
         }
 
-        public static void SetCodexDockerImageProvider(this CoreInterface ci, ICodexDockerImageProvider provider)
-        {
-            Plugin(ci).SetCodexDockerImageProvider(provider);
-        }
-
         private static CodexContractsPlugin Plugin(CoreInterface ci)
         {
             return ci.GetPlugin<CodexContractsPlugin>();

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,5 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using BlockchainUtils;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Newtonsoft.Json;
 using Utils;
 
@@ -15,6 +16,11 @@ namespace CodexContractsPlugin.Marketplace
         byte[] RequestId { get; set; }
     }
 
+    public interface IHasSlotIndex
+    {
+        ulong SlotIndex { get; set; }
+    }
+
     public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
@@ -51,20 +57,25 @@ namespace CodexContractsPlugin.Marketplace
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
+
+        public override string ToString()
+        {
+            return $"SlotFilled:[host:{Host} request:{RequestId.ToHex()} slotIndex:{SlotIndex}]";
+        }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -75,5 +86,11 @@ namespace CodexContractsPlugin.Marketplace
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
+
+    public partial class ReserveSlotFunction : IHasBlock, IHasRequestId, IHasSlotIndex
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/VersionRegistry.cs

@@ -1,125 +0,0 @@
-using System.Diagnostics;
-using Logging;
-
-namespace CodexContractsPlugin
-{
-    public interface ICodexDockerImageProvider
-    {
-        string GetCodexDockerImage();
-    }
-
-    public class VersionRegistry
-    {
-        private ICodexDockerImageProvider provider = new ExceptionProvider();
-        private static readonly Dictionary<string, string> cache = new Dictionary<string, string>();
-        private static readonly object cacheLock = new object();
-        private readonly ILog log;
-
-        public VersionRegistry(ILog log)
-        {
-            this.log = log;
-        }
-
-        public void SetProvider(ICodexDockerImageProvider provider)
-        {
-            this.provider = provider;
-        }
-
-        public string GetContractsDockerImage()
-        {
-            try
-            {
-                var codexImage = provider.GetCodexDockerImage();
-                return GetContractsDockerImage(codexImage);
-            }
-            catch (Exception exc)
-            {
-                throw new Exception("Failed to get contracts docker image.", exc);
-            }
-        }
-
-        private string GetContractsDockerImage(string codexImage)
-        {
-            lock (cacheLock)
-            {
-                if (cache.TryGetValue(codexImage, out string? value))
-                {
-                    return value;
-                }
-                var result = GetContractsImage(codexImage);
-                cache.Add(codexImage, result);
-                return result;
-            }
-        }
-
-        private string GetContractsImage(string codexImage)
-        {
-            var inspectResult = InspectCodexImage(codexImage);
-            var image = ParseCodexContractsImageName(inspectResult);
-            log.Log($"From codex docker image '{codexImage}', determined codex-contracts docker image: '{image}'");
-            return image;
-        }
-
-        private string InspectCodexImage(string img)
-        {
-            Execute("docker", $"pull {img}");
-            return Execute("docker", $"inspect {img}");
-        }
-
-        private string ParseCodexContractsImageName(string inspectResult)
-        {
-            // It is a nice json structure. But we only need this one line.
-            // "storage.codex.nim-codex.blockchain-image": "codexstorage/codex-contracts-eth:sha-0bf1385-dist-tests"
-            var lines = inspectResult.Split('\n', StringSplitOptions.RemoveEmptyEntries);
-            var line = lines.Single(l => l.Contains("storage.codex.nim-codex.blockchain-image"));
-            var tokens = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
-            return tokens.Last().Replace("\"", "").Trim();
-        }
-
-        private string Execute(string cmd, string args)
-        {
-            var startInfo = new ProcessStartInfo(
-                fileName: cmd,
-                arguments: args
-            );
-            startInfo.RedirectStandardOutput = true;
-            startInfo.RedirectStandardError = true;
-
-            var process = Process.Start(startInfo);
-            if (process == null)
-            {
-                throw new Exception("Failed to start: " + cmd + args);
-            }
-            KillAfterTimeout(process);
-
-            process.WaitForExit();
-            return process.StandardOutput.ReadToEnd();
-        }
-
-        private void KillAfterTimeout(Process process)
-        {
-            // There's a known issue that some docker commands on some platforms
-            // will fail to stop on their own. This has been known since 2019 and it's not fixed.
-            // So we will issue a kill to the process ourselves if it exceeds a timeout.
-
-            Task.Run(() =>
-            {
-                Thread.Sleep(TimeSpan.FromSeconds(30.0));
-
-                if (process != null && !process.HasExited)
-                {
-                    process.Kill();
-                }
-            });
-        }
-    }
-
-    internal class ExceptionProvider : ICodexDockerImageProvider
-    {
-        public string GetCodexDockerImage()
-        {
-            throw new InvalidOperationException("CodexContractsPlugin has not yet received a CodexDockerImageProvider " +
-                "and so cannot select a compatible contracts docker image.");
-        }
-    }
-}

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "1A-F7-DF-C3-E1-C6-98-FF-32-20-21-9B-26-40-B0-51-08-35-C2-E7-DB-41-49-93-60-A9-CE-47-B5-AD-3D-A3";
+        private const string OpenApiYamlHash = "2F-9D-82-3C-F0-2F-D3-C9-72-C3-F2-6E-BD-C3-63-F5-67-62-D1-03-B6-60-75-31-22-DF-3F-63-A2-8D-AA-4B";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexDockerImage.cs

@@ -1,10 +1,8 @@
-using CodexContractsPlugin;
-
-namespace CodexPlugin
+namespace CodexPlugin
 {
-    public class CodexDockerImage : ICodexDockerImageProvider
+    public class CodexDockerImage
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:0.2.3-dist-tests";
 
         public static string Override { get; set; } = string.Empty;
 

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -42,7 +42,6 @@ namespace CodexPlugin
 
         public void Awake(IPluginAccess access)
         {
-            access.GetPlugin<CodexContractsPlugin.CodexContractsPlugin>().SetCodexDockerImageProvider(codexDockerImage);
         }
 
         public void Announce()

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

@@ -23,16 +23,27 @@ namespace CodexPlugin.OverwatchSupport
             converter = new CodexLogConverter(writer, config, identityMap);
         }
 
-        public void Finalize(string outputFilepath)
+        public void FinalizeWriter()
         {
             log.Log("Finalizing Codex transcript...");
 
             writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
-            writer.Write(outputFilepath);
+            writer.Write(GetOutputFullPath());
 
             log.Log("Done");
         }
 
+        private string GetOutputFullPath()
+        {
+            var outputPath = Path.GetDirectoryName(log.GetFullName());
+            if (outputPath == null) throw new Exception("Logfile path is null");
+            var filename = Path.GetFileNameWithoutExtension(log.GetFullName());
+            if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty");
+            var outputFile = Path.Combine(outputPath, filename + "_" + config.OutputPath);
+            if (!outputFile.EndsWith(".owts")) outputFile += ".owts";
+            return outputFile;
+        }
+
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
             nodeName = Str.Between(nodeName, "'", "'");

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs

@@ -2,11 +2,13 @@
 {
     public class CodexTranscriptWriterConfig
     {
-        public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents)
+        public CodexTranscriptWriterConfig(string outputPath, bool includeBlockReceivedEvents)
         {
+            OutputPath = outputPath;
             IncludeBlockReceivedEvents = includeBlockReceivedEvents;
         }
 
+        public string OutputPath { get; }
         public bool IncludeBlockReceivedEvents { get; }
     }
 }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -31,6 +31,7 @@ namespace GethPlugin
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
         BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
         BlockTimeEntry GetBlockForNumber(ulong number);
+        void IterateFunctionCalls<TFunc>(BlockInterval blockInterval, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new();
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -183,6 +184,33 @@ namespace GethPlugin
             return StartInteraction().GetBlockForNumber(number);
         }
 
+        public BlockWithTransactions GetBlk(ulong number)
+        {
+            return StartInteraction().GetBlockWithTransactions(number);
+        }
+
+        public void IterateFunctionCalls<TFunc>(BlockInterval blockRange, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new()
+        {
+            var i = StartInteraction();
+            for (var blkI = blockRange.From; blkI <= blockRange.To; blkI++)
+            {
+                var blk = i.GetBlockWithTransactions(blkI);
+
+                foreach (var t in blk.Transactions)
+                {
+                    if (t.IsTransactionForFunctionMessage<TFunc>())
+                    {
+                        var func = t.DecodeTransactionToFunctionMessage<TFunc>();
+                        if (func != null)
+                        {
+                            var b = GetBlockForNumber(blkI);
+                            onCall(b, func);
+                        }
+                    }
+                }
+            }
+        }
+
         protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/MetricsPlugin/MetricsDownloader.cs

@@ -28,11 +28,11 @@ namespace MetricsPlugin
             var file = log.CreateSubfile("csv");
             log.Log($"Downloading metrics for {nodeName} to file {file.Filename}");
 
-            file.WriteRaw(string.Join(",", headers));
+            file.Write(string.Join(",", headers));
 
             foreach (var pair in map)
             {
-                file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
+                file.Write(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
             }
 
             return file;

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -199,7 +199,7 @@ namespace ContinuousTests
 
             private void WriteEntryToFile(LogQueueEntry currentEntry)
             {
-                targetFile.WriteRaw(currentEntry.Message);
+                targetFile.Write(currentEntry.Message);
             }
 
             private void DeleteOldEntries(ulong wantedNumber)

Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs

@@ -50,8 +50,9 @@ namespace CodexReleaseTests.DataTests
             var blockTtl = TimeSpan.FromMinutes(1.0);
             var interval = TimeSpan.FromSeconds(10.0);
 
+            var bootstrapNode = StartCodex();
             var geth = StartGethNode(s => s.IsMiner());
-            var contracts = Ci.StartCodexContracts(geth);
+            var contracts = Ci.StartCodexContracts(geth, bootstrapNode.Version);
             var node = StartCodex(s => s
                 .EnableMarketplace(geth, contracts, m => m.WithInitial(100.Eth(), 100.Tst()))
                 .WithBlockTTL(blockTtl)

Tests/CodexReleaseTests/DataTests/DecodeTest.cs

@@ -1,5 +1,5 @@
 using System.Security.Cryptography;
-using CodexReleaseTests.MarketTests;
+using CodexReleaseTests.Utils;
 using Nethereum.JsonRpc.Client;
 using NUnit.Framework;
 using Utils;

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -1,173 +0,0 @@
-using CodexClient;
-using CodexContractsPlugin.ChainMonitor;
-using CodexContractsPlugin.Marketplace;
-using CodexPlugin;
-using NUnit.Framework;
-using System.Numerics;
-using Utils;
-
-namespace CodexReleaseTests.MarketTests
-{
-    public class ContractFailedTest : MarketplaceAutoBootstrapDistTest
-    {
-        private const int FilesizeMb = 10;
-        private const int NumberOfSlots = 3;
-
-        protected override int NumberOfHosts => 6;
-        protected override int NumberOfClients => 1;
-        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
-        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
-        private readonly TestToken pricePerBytePerSecond = 10.TstWei();
-
-        [Test]
-        public void ContractFailed()
-        {
-            var hosts = StartHosts();
-            var client = StartClients().Single();
-            var validator = StartValidator();
-
-            var request = CreateStorageRequest(client);
-
-            request.WaitForStorageContractSubmitted();
-            AssertContractIsOnChain(request);
-
-            request.WaitForStorageContractStarted();
-            AssertContractSlotsAreFilledByHosts(request, hosts);
-
-            hosts.Stop(waitTillStopped: true);
-            
-            var config = GetContracts().Deployment.Config;
-            request.WaitForContractFailed(config);
-
-            var frees = GetOnChainSlotFrees(hosts);
-            Assert.That(frees.Length, Is.EqualTo(
-                request.Purchase.MinRequiredNumberOfNodes - request.Purchase.NodeFailureTolerance));
-
-            var periodReports = GetPeriodMonitorReports();
-            var missedProofs = periodReports.Reports.SelectMany(r => r.MissedProofs).ToArray();
-            AssertEnoughProofsWereMissedForSlotFree(frees, missedProofs, config);
-
-            AssertClientPaidNothing(client);
-            AssertValidatorWasPaidPerMissedProof(validator, request, missedProofs, config);
-            AssertHostCollateralWasBurned(hosts, request);
-        }
-
-        private void AssertClientPaidNothing(ICodexNode client)
-        {
-            AssertTstBalance(client, StartingBalanceTST.Tst(), "Client should not have paid for failed contract.");
-        }
-
-        private void AssertValidatorWasPaidPerMissedProof(ICodexNode validator, IStoragePurchaseContract request, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
-        {
-            var rewardPerMissedProof = GetValidatorRewardPerMissedProof(request, config);
-            var totalValidatorReward = rewardPerMissedProof * missedProofs.Length;
-
-            AssertTstBalance(validator, StartingBalanceTST.Tst() + totalValidatorReward, $"Validator is rewarded per slot marked as missing. " +
-                $"numberOfMissedProofs: {missedProofs.Length} rewardPerMissedProof: {rewardPerMissedProof}");
-        }
-
-        private TestToken GetCollatoralPerSlot(IStoragePurchaseContract request)
-        {
-            var slotSize = new ByteSize(Convert.ToInt64(request.GetStatus()!.Request.Ask.SlotSize));
-            return new TestToken(request.Purchase.CollateralPerByte.TstWei * slotSize.SizeInBytes);
-        }
-
-        private void AssertHostCollateralWasBurned(ICodexNodeGroup hosts, IStoragePurchaseContract request)
-        {
-            var slotFills = GetOnChainSlotFills(hosts);
-            foreach (var host in hosts)
-            {
-                AssertHostCollateralWasBurned(host, slotFills, request);
-            }
-        }
-
-        private void AssertHostCollateralWasBurned(ICodexNode host, SlotFill[] slotFills, IStoragePurchaseContract request)
-        {
-            // In case of a failed contract, the entire slotColateral is lost.
-            var filledByHost = slotFills.Where(f => f.Host.EthAddress == host.EthAddress).ToArray();
-            var numSlotsOfHost = filledByHost.Length;
-            var collatoralPerSlot = GetCollatoralPerSlot(request);
-            var totalCost = collatoralPerSlot * numSlotsOfHost;
-
-            AssertTstBalance(host, StartingBalanceTST.Tst() - totalCost, $"Host has lost collateral for each slot. " +
-                $"numberOfSlotsByHost: {numSlotsOfHost} collateralPerSlot: {collatoralPerSlot}");
-        }
-
-        private TestToken GetValidatorRewardPerMissedProof(IStoragePurchaseContract request, MarketplaceConfig config)
-        {
-            var collatoralPerSlot = GetCollatoralPerSlot(request);
-            var slashPercentage = config.Collateral.SlashPercentage;
-            var validatorRewardPercentage = config.Collateral.ValidatorRewardPercentage;
-
-            var rewardPerMissedProof =
-                PercentageOf(
-                    PercentageOf(collatoralPerSlot, slashPercentage),
-                    validatorRewardPercentage);
-
-            return rewardPerMissedProof;
-        }
-
-        private TestToken PercentageOf(TestToken value, byte percentage)
-        {
-            var p = new BigInteger(percentage);
-            return new TestToken((value.TstWei * p) / 100);
-        }
-
-        private void AssertEnoughProofsWereMissedForSlotFree(SlotFree[] frees, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
-        {
-            foreach (var free in frees)
-            {
-                AssertEnoughProofsWereMissedForSlotFree(free, missedProofs, config);
-            }
-        }
-
-        private void AssertEnoughProofsWereMissedForSlotFree(SlotFree free, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
-        {
-            var missedByHost = missedProofs.Where(p => p.Host != null && p.Host.Address == free.Host.EthAddress.Address).ToArray();
-            var maxNumMissedProofsBeforeFreeSlot = config.Collateral.MaxNumberOfSlashes;
-            Assert.That(missedByHost.Length, Is.EqualTo(maxNumMissedProofsBeforeFreeSlot));
-        }
-
-        private TimeSpan CalculateContractFailTimespan()
-        {
-            var config = GetContracts().Deployment.Config;
-            var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
-            var periodDuration = GetPeriodDuration();
-
-            // Each host could miss 1 proof per period,
-            // so the time we should wait is period time * requiredNum of missed proofs.
-            // Except: the proof requirement has a concept of "downtime":
-            // a segment of time where proof is not required.
-            // We calculate the probability of downtime and extend the waiting
-            // timeframe by a factor, such that all hosts are highly likely to have 
-            // failed a sufficient number of proofs.
-
-            float n = requiredNumMissedProofs;
-            return periodDuration * n * GetDowntimeFactor(config);
-        }
-
-        private float GetDowntimeFactor(MarketplaceConfig config)
-        {
-            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
-            float downtime = numBlocksInDowntimeSegment;
-            float window = 256.0f;
-            var chanceOfDowntime = downtime / window;
-            return 1.0f + chanceOfDowntime + chanceOfDowntime;
-        }
-
-        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
-        {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
-            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
-            {
-                Duration = TimeSpan.FromMinutes(20.0),
-                Expiry = TimeSpan.FromMinutes(5.0),
-                MinRequiredNumberOfNodes = NumberOfSlots,
-                NodeFailureTolerance = 1,
-                PricePerBytePerSecond = pricePerBytePerSecond,
-                ProofProbability = 1, // Require a proof every period
-                CollateralPerByte = 1.TstWei()
-            });
-        }
-    }
-}

Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs

@@ -1,18 +0,0 @@
-using NUnit.Framework;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace CodexReleaseTests.MarketTests
-{
-    public class ContractRepairedTest
-    {
-        [Test]
-        [Ignore("TODO - Test in which a host fails, but the slot is repaired")]
-        public void ContractRepaired()
-        {
-        }
-    }
-}

Tests/CodexReleaseTests/MarketTests/FailTest.cs

@@ -0,0 +1,78 @@
+using CodexClient;
+using CodexReleaseTests.Utils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class FailTest : MarketplaceAutoBootstrapDistTest
+    {
+        protected override int NumberOfHosts => 4;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => 1.GB();
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(1.0);
+
+        [Ignore("Slots are never freed because proofs are never marked as missing. Issue: https://github.com/codex-storage/nim-codex/issues/1153")]
+        [Test]
+        [Combinatorial]
+        public void Fail(
+            [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun
+        )
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+            StartValidator();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            hosts.Stop(waitTillStopped: true);
+
+            WaitForSlotFreedEvents();
+
+            var config = GetContracts().Deployment.Config;
+            request.WaitForContractFailed(config);
+        }
+
+        private void WaitForSlotFreedEvents()
+        {
+            var start = DateTime.UtcNow;
+            var timeout = CalculateContractFailTimespan();
+
+            Log($"{nameof(WaitForSlotFreedEvents)} timeout: {Time.FormatDuration(timeout)}");
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var slotFreed = events.GetSlotFreedEvents();
+                if (slotFreed.Length == NumberOfHosts)
+                {
+                    Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
+                    return;
+                }
+                GetContracts().WaitUntilNextPeriod();
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(5.MB()));
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = HostAvailabilityMaxDuration / 2,
+                Expiry = TimeSpan.FromMinutes(5.0),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerBytePerSecond = 100.TstWei(),
+                ProofProbability = 1, // Require a proof every period
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/FinishTest.cs

@@ -1,22 +1,34 @@
 using CodexClient;
+using CodexReleaseTests.Utils;
 using NUnit.Framework;
 using Utils;
 
 namespace CodexReleaseTests.MarketTests
 {
-    [TestFixture]
-    public class ContractSuccessfulTest : MarketplaceAutoBootstrapDistTest
+    [TestFixture(5, 3, 1)]
+    [TestFixture(10, 20, 10)]
+    public class FinishTest : MarketplaceAutoBootstrapDistTest
     {
-        private const int FilesizeMb = 10;
+        public FinishTest(int hosts, int slots, int tolerance)
+        {
+            this.hosts = hosts;
+            purchaseParams = new PurchaseParams(slots, tolerance, uploadFilesize: 10.MB());
+        }
 
-        protected override int NumberOfHosts => 6;
-        protected override int NumberOfClients => 1;
-        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
-        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
         private readonly TestToken pricePerBytePerSecond = 10.TstWei();
+        private readonly int hosts;
+        private readonly PurchaseParams purchaseParams;
+
+        protected override int NumberOfHosts => hosts;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(5.1);
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
 
         [Test]
-        public void ContractSuccessful()
+        [Combinatorial]
+        public void Finish(
+            [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun
+        )
         {
             var hosts = StartHosts();
             var client = StartClients().Single();
@@ -27,7 +39,7 @@ namespace CodexReleaseTests.MarketTests
             request.WaitForStorageContractSubmitted();
             AssertContractIsOnChain(request);
 
-            request.WaitForStorageContractStarted();
+            WaitForContractStarted(request);
             AssertContractSlotsAreFilledByHosts(request, hosts);
 
             request.WaitForStorageContractFinished();
@@ -40,16 +52,14 @@ namespace CodexReleaseTests.MarketTests
 
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 Duration = GetContractDuration(),
                 Expiry = GetContractExpiry(),
-                // TODO: this should work with NumberOfHosts, but
-                // an ongoing issue makes hosts sometimes not pick up slots.
-                // When it's resolved, we can reduce the number of hosts and slim down this test.
-                MinRequiredNumberOfNodes = 3,
-                NodeFailureTolerance = 1,
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
                 PricePerBytePerSecond = pricePerBytePerSecond,
                 ProofProbability = 20,
                 CollateralPerByte = 100.TstWei()
@@ -63,7 +73,7 @@ namespace CodexReleaseTests.MarketTests
 
         private TimeSpan GetContractDuration()
         {
-            return Get8TimesConfiguredPeriodDuration() / 2;
+            return Get8TimesConfiguredPeriodDuration();
         }
 
         private TimeSpan Get8TimesConfiguredPeriodDuration()

Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs

@@ -1,89 +0,0 @@
-using CodexClient;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexReleaseTests.MarketTests
-{
-    [TestFixture]
-    public class MultipleContractsTest : MarketplaceAutoBootstrapDistTest
-    {
-        private const int FilesizeMb = 10;
-
-        protected override int NumberOfHosts => 8;
-        protected override int NumberOfClients => 3;
-        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
-        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
-        private readonly TestToken pricePerBytePerSecond = 10.TstWei();
-
-        [Test]
-        [Ignore("TODO - Test where multiple successful contracts are run simultaenously")]
-        public void MultipleSuccessfulContracts()
-        {
-            var hosts = StartHosts();
-            var clients = StartClients();
-
-            var requests = clients.Select(c => CreateStorageRequest(c)).ToArray();
-
-            All(requests, r =>
-            {
-                r.WaitForStorageContractSubmitted();
-                AssertContractIsOnChain(r);
-            });
-
-            All(requests, r => r.WaitForStorageContractStarted());
-            All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts));
-
-            All(requests, r => r.WaitForStorageContractFinished());
-
-                // todo: removed from codexclient:
-                //contracts.WaitUntilNextPeriod();
-                //contracts.WaitUntilNextPeriod();
-
-                //var blocks = 3;
-                //Log($"Waiting {blocks} blocks for nodes to process payouts...");
-                //Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
-
-            // todo:
-            //AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
-            //AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
-            //AssertHostsCollateralsAreUnchanged(hosts);
-        }
-
-        private void All(IStoragePurchaseContract[] requests, Action<IStoragePurchaseContract> action)
-        {
-            foreach (var r in requests) action(r);
-        }
-
-        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
-        {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
-            var config = GetContracts().Deployment.Config;
-            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
-            {
-                Duration = GetContractDuration(),
-                Expiry = GetContractExpiry(),
-                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
-                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
-                PricePerBytePerSecond = pricePerBytePerSecond,
-                ProofProbability = 20,
-                CollateralPerByte = 1.Tst()
-            });
-        }
-
-        private TimeSpan GetContractExpiry()
-        {
-            return GetContractDuration() / 2;
-        }
-
-        private TimeSpan GetContractDuration()
-        {
-            return Get8TimesConfiguredPeriodDuration() / 2;
-        }
-
-        private TimeSpan Get8TimesConfiguredPeriodDuration()
-        {
-            var config = GetContracts().Deployment.Config;
-            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
-        }
-    }
-}

Tests/CodexReleaseTests/MarketTests/RepairTest.cs

@@ -0,0 +1,177 @@
+using CodexClient;
+using CodexReleaseTests.Utils;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class RepairTest : MarketplaceAutoBootstrapDistTest
+    {
+        #region Setup
+
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 4,
+            tolerance: 2,
+            uploadFilesize: 32.MB()
+        );
+
+        public RepairTest()
+        {
+            Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
+        }
+
+        protected override int NumberOfHosts => 5;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
+
+        #endregion
+
+        [Ignore("Test is ready. Waiting for repair implementation. " +
+            "Slots are never freed because proofs are never marked as missing. Issue: https://github.com/codex-storage/nim-codex/issues/1153")]
+        [Test]
+        [Combinatorial]
+        public void RollingRepairSingleFailure(
+            [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun,
+            [Values(10)] int numFailures)
+        {
+            var hosts = StartHosts().ToList();
+            var client = StartClients().Single();
+            StartValidator();
+
+            var contract = CreateStorageRequest(client);
+            contract.WaitForStorageContractStarted();
+            // All slots are filled.
+
+            for (var i = 0; i < numFailures; i++)
+            {
+                Log($"Failure step: {i}");
+
+                // Start a new host. Add it to the back of the list:
+                hosts.Add(StartOneHost());
+
+                var fill = GetSlotFillByOldestHost(hosts);
+
+                Log($"Causing failure for host: {fill.Host.GetName()} slotIndex: {fill.SlotFilledEvent.SlotIndex}");
+                hosts.Remove(fill.Host);
+                fill.Host.Stop(waitTillStopped: true);
+
+                // The slot should become free.
+                WaitForSlotFreedEvent(contract, fill.SlotFilledEvent.SlotIndex);
+
+                // One of the other hosts should pick up the free slot.
+                WaitForNewSlotFilledEvent(contract, fill.SlotFilledEvent.SlotIndex);
+            }
+        }
+
+        private void WaitForSlotFreedEvent(IStoragePurchaseContract contract, ulong slotIndex)
+        {
+            Log(nameof(WaitForSlotFreedEvent));
+            var start = DateTime.UtcNow;
+            var timeout = CalculateContractFailTimespan();
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var slotsFreed = events.GetSlotFreedEvents();
+                Log($"Slots freed this period: {slotsFreed.Length}");
+
+                foreach (var free in slotsFreed)
+                {
+                    if (free.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant())
+                    {
+                        if (free.SlotIndex == slotIndex)
+                        {
+                            Log("Found the correct slotFree event");
+                            return;
+                        }
+                    }
+                }
+
+                GetContracts().WaitUntilNextPeriod();
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private void WaitForNewSlotFilledEvent(IStoragePurchaseContract contract, ulong slotIndex)
+        {
+            Log(nameof(WaitForNewSlotFilledEvent));
+            var start = DateTime.UtcNow;
+            var timeout = contract.Purchase.Expiry;
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var newTimeRange = new TimeRange(start, DateTime.UtcNow); // We only want to see new fill events.
+                var events = GetContracts().GetEvents(newTimeRange);
+                var slotFillEvents = events.GetSlotFilledEvents();
+
+                var matches = slotFillEvents.Where(f =>
+                {
+                    return
+                        f.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant() &&
+                        f.SlotIndex == slotIndex;
+                }).ToArray();
+
+                if (matches.Length > 1)
+                {
+                    var msg = string.Join(",", matches.Select(f => f.ToString()));
+                    Assert.Fail($"Somehow, the slot got filled multiple times: {msg}");
+                }
+                if (matches.Length == 1)
+                {
+                    Log($"Found the correct new slotFilled event: {matches[0].ToString()}");
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(15));
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private SlotFill GetSlotFillByOldestHost(List<ICodexNode> hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var copy = hosts.ToArray();
+            foreach (var host in copy)
+            {
+                var fill = GetFillByHost(host, fills);
+                if (fill == null)
+                {
+                    // This host didn't fill anything.
+                    // Move this one to the back of the list.
+                    hosts.Remove(host);
+                    hosts.Add(host);
+                }
+                else
+                {
+                    return fill;
+                }
+            }
+            throw new Exception("None of the hosts seem to have filled a slot.");
+        }
+
+        private SlotFill? GetFillByHost(ICodexNode host, SlotFill[] fills)
+        {
+            // If these is more than 1 fill by this host, the test is misconfigured.
+            // The availability size of the host should guarantee it can fill 1 slot maximum.
+            return fills.SingleOrDefault(f => f.Host.EthAddress == host.EthAddress);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = HostAvailabilityMaxDuration / 2,
+                Expiry = TimeSpan.FromMinutes(10.0),
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
+                PricePerBytePerSecond = 10.TstWei(),
+                ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs

@@ -0,0 +1,117 @@
+using CodexClient;
+using CodexPlugin;
+using CodexReleaseTests.Utils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture(10, 20, 5)]
+    public class SequentialContracts : MarketplaceAutoBootstrapDistTest
+    {
+        public SequentialContracts(int hosts, int slots, int tolerance)
+        {
+            this.hosts = hosts;
+            purchaseParams = new PurchaseParams(slots, tolerance, 10.MB());
+        }
+
+        private readonly int hosts;
+        private readonly PurchaseParams purchaseParams;
+
+        protected override int NumberOfHosts => hosts;
+        protected override int NumberOfClients => 6;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(100.0);
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
+        private readonly TestToken pricePerBytePerSecond = 10.TstWei();
+
+        [Test]
+        [Combinatorial]
+        public void Sequential(
+            [Values(10)] int numGenerations)
+        {
+            var hosts = StartHosts();
+            var clients = StartClients();
+
+            for (var i = 0; i < numGenerations; i++)
+            {
+                Log("Generation: " + i);
+                try
+                {
+                    Generation(clients, hosts);
+                }
+                catch (Exception ex)
+                {
+                    Assert.Fail($"Failed at generation {i} with exception {ex}");
+                }
+            }
+
+            Thread.Sleep(TimeSpan.FromSeconds(12.0));
+        }
+
+        private void Generation(ICodexNodeGroup clients, ICodexNodeGroup hosts)
+        {
+            var requests = All(clients.ToArray(), CreateStorageRequest);
+
+            All(requests, r =>
+            {
+                r.WaitForStorageContractSubmitted();
+                AssertContractIsOnChain(r);
+            });
+
+            All(requests, WaitForContractStarted);
+        }
+
+        private void All<T>(T[] items, Action<T> action)
+        {
+            var tasks = items.Select(r => Task.Run(() => action(r))).ToArray();
+            Task.WaitAll(tasks);
+            foreach(var t in tasks)
+            {
+                if (t.Exception != null) throw t.Exception;
+            }
+        }
+
+        private TResult[] All<T, TResult>(T[] items, Func<T, TResult> action)
+        {
+            var tasks = items.Select(r => Task.Run(() => action(r))).ToArray();
+            Task.WaitAll(tasks);
+            foreach (var t in tasks)
+            {
+                if (t.Exception != null) throw t.Exception;
+            }
+            return tasks.Select(t => t.Result).ToArray();
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
+                PricePerBytePerSecond = pricePerBytePerSecond,
+                ProofProbability = 10000,
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() * 4;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(config.Proofs.Period * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/StartTest.cs

@@ -0,0 +1,72 @@
+using CodexClient;
+using CodexReleaseTests.Utils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class StartTest : MarketplaceAutoBootstrapDistTest
+    {
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 3,
+            tolerance: 1,
+            uploadFilesize: 10.MB()
+        );
+        private readonly TestToken pricePerBytePerSecond = 10.TstWei();
+
+        protected override int NumberOfHosts => 5;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(10.0);
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
+
+        [Test]
+        [Combinatorial]
+        public void Start(
+            [Values([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])] int rerun
+        )
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            WaitForContractStarted(request);
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
+                PricePerBytePerSecond = pricePerBytePerSecond,
+                ProofProbability = 20,
+                CollateralPerByte = 100.TstWei()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration();
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            return GetPeriodDuration() * 8.0;
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs

@@ -1,11 +1,5 @@
-using CodexPlugin;
-using CodexTests;
+using CodexTests;
 using NUnit.Framework;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using Utils;
 
 namespace CodexReleaseTests.NodeTests

Tests/CodexReleaseTests/Parallelism.cs

@@ -1,6 +1,6 @@
 using NUnit.Framework;
 
-[assembly: LevelOfParallelism(1)]
-namespace CodexReleaseTests.DataTests
+[assembly: LevelOfParallelism(2)]
+namespace CodexReleaseTests
 {
 }

Tests/CodexReleaseTests/Utils/ChainMonitor.cs

@@ -0,0 +1,79 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
+using Logging;
+
+namespace CodexReleaseTests.Utils
+{
+    public class ChainMonitor
+    {
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly DateTime startUtc;
+        private readonly TimeSpan updateInterval;
+        private CancellationTokenSource cts = new CancellationTokenSource();
+        private Task worker = Task.CompletedTask;
+
+        public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc)
+            : this(log, contracts, startUtc, TimeSpan.FromSeconds(3.0))
+        {
+        }
+
+        public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval)
+        {
+            this.log = log;
+            this.contracts = contracts;
+            this.startUtc = startUtc;
+            this.updateInterval = updateInterval;
+        }
+
+        public void Start(Action onFailure)
+        {
+            cts = new CancellationTokenSource();
+            worker = Task.Run(() => Worker(onFailure));
+        }
+
+        public void Stop()
+        {
+            cts.Cancel();
+            worker.Wait();
+            if (worker.Exception != null) throw worker.Exception;
+        }
+
+        private void Worker(Action onFailure)
+        {
+            var state = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, doProofPeriodMonitoring: true);
+            Thread.Sleep(updateInterval);
+
+            log.Log("Chain monitoring started");
+            while (!cts.IsCancellationRequested)
+            {
+                try
+                {
+                    UpdateChainState(state);
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Exception in chain monitor: " + ex);
+                    onFailure();
+                    throw;
+                }
+
+                cts.Token.WaitHandle.WaitOne(updateInterval);
+            }
+        }
+
+        private void UpdateChainState(ChainState state)
+        {
+            state.Update();
+
+            var reports = state.PeriodMonitor.GetAndClearReports();
+            if (reports.IsEmpty) return;
+
+            var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots));
+            var required = reports.Reports.Sum(r => Convert.ToInt32(r.TotalProofsRequired));
+            var missed = reports.Reports.Sum(r => r.MissedProofs.Length);
+
+            log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}");
+        }
+    }
+}

Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs

@@ -1,49 +1,45 @@
 using CodexClient;
 using CodexContractsPlugin;
-using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
 using CodexTests;
-using DistTestCore;
 using GethPlugin;
+using Logging;
 using Nethereum.Hex.HexConvertors.Extensions;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexReleaseTests.MarketTests
+namespace CodexReleaseTests.Utils
 {
     public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
     {
-        private readonly Dictionary<TestLifecycle, MarketplaceHandle> handles = new Dictionary<TestLifecycle, MarketplaceHandle>();
+        private MarketplaceHandle handle = null!;
         protected const int StartingBalanceTST = 1000;
         protected const int StartingBalanceEth = 10;
 
-        protected override void LifecycleStart(TestLifecycle lifecycle)
+        [SetUp]
+        public void SetupMarketplace()
         {
-            base.LifecycleStart(lifecycle);
             var geth = StartGethNode(s => s.IsMiner());
-            var contracts = Ci.StartCodexContracts(geth);
-            var monitor = new ChainMonitor(lifecycle.Log, contracts, lifecycle.TestStart, TimeSpan.FromSeconds(1.0));
-            monitor.Start();
-
-            handles.Add(lifecycle, new MarketplaceHandle(geth, contracts, monitor));
+            var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
+            var monitor = SetupChainMonitor(GetTestLog(), contracts, GetTestRunTimeRange().From);
+            handle = new MarketplaceHandle(geth, contracts, monitor);
         }
 
-        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
+        [TearDown]
+        public void TearDownMarketplace()
         {
-            handles[lifecycle].Monitor.Stop();
-            handles.Remove(lifecycle);
-            base.LifecycleStop(lifecycle, result);
+            if (handle.ChainMonitor != null) handle.ChainMonitor.Stop();
         }
 
         protected IGethNode GetGeth()
         {
-            return handles[Get()].Geth;
+            return handle.Geth;
         }
 
         protected ICodexContracts GetContracts()
         {
-            return handles[Get()].Contracts;
+            return handle.Contracts;
         }
 
         protected TimeSpan GetPeriodDuration()
@@ -52,24 +48,16 @@ namespace CodexReleaseTests.MarketTests
             return TimeSpan.FromSeconds(config.Proofs.Period);
         }
 
-        protected PeriodMonitorResult GetPeriodMonitorReports()
-        {
-            return handles[Get()].Monitor.GetPeriodReports();
-        }
-
         protected abstract int NumberOfHosts { get; }
         protected abstract int NumberOfClients { get; }
         protected abstract ByteSize HostAvailabilitySize { get; }
         protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
-        protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0);
+        protected virtual bool MonitorChainState { get; } = true;
 
         public ICodexNodeGroup StartHosts()
         {
             var hosts = StartCodex(NumberOfHosts, s => s
                 .WithName("host")
-                .WithBlockTTL(HostBlockTTL)
-                .WithBlockMaintenanceNumber(100)
-                .WithBlockMaintenanceInterval(HostBlockTTL / 2)
                 .EnableMarketplace(GetGeth(), GetContracts(), m => m
                     .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
                     .AsStorageNode()
@@ -81,47 +69,38 @@ namespace CodexReleaseTests.MarketTests
             {
                 AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartHosts));
                 AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartHosts));
-
-                var spaceBefore = host.Space();
-                Assert.That(spaceBefore.QuotaReservedBytes, Is.EqualTo(0));
-                Assert.That(spaceBefore.QuotaUsedBytes, Is.EqualTo(0));
-
-                host.Marketplace.MakeStorageAvailable(new CreateStorageAvailability(
+                
+                host.Marketplace.MakeStorageAvailable(new StorageAvailability(
                     totalSpace: HostAvailabilitySize,
                     maxDuration: HostAvailabilityMaxDuration,
                     minPricePerBytePerSecond: 1.TstWei(),
                     totalCollateral: 999999.Tst())
                 );
-
-                var spaceAfter = host.Space();
-                Assert.That(spaceAfter.QuotaReservedBytes, Is.EqualTo(HostAvailabilitySize.SizeInBytes));
-                Assert.That(spaceAfter.QuotaUsedBytes, Is.EqualTo(0));
             }
             return hosts;
         }
 
-        public void AssertHostAvailabilitiesAreEmpty(IEnumerable<ICodexNode> hosts)
+        public ICodexNode StartOneHost()
         {
-            var retry = GetAvailabilitySpaceAssertRetry();
-            retry.Run(() =>
-            {
-                foreach (var host in hosts)
-                {
-                    AssertHostAvailabilitiesAreEmpty(host);
-                }
-            });
-        }
+            var host = StartCodex(s => s
+                .WithName("singlehost")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsStorageNode()
+                )
+            );
 
-        private void AssertHostAvailabilitiesAreEmpty(ICodexNode host)
-        {
-            var availabilities = host.Marketplace.GetAvailabilities();
-            foreach (var a in availabilities)
-            {
-                if (a.FreeSpace.SizeInBytes  != a.TotalSpace.SizeInBytes)
-                {
-                    throw new Exception(nameof(AssertHostAvailabilitiesAreEmpty) + $" free: {a.FreeSpace} total: {a.TotalSpace}");
-                }
-            }
+            var config = GetContracts().Deployment.Config;
+            AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartOneHost));
+            AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartOneHost));
+
+            host.Marketplace.MakeStorageAvailable(new StorageAvailability(
+                totalSpace: HostAvailabilitySize,
+                maxDuration: HostAvailabilityMaxDuration,
+                minPricePerBytePerSecond: 1.TstWei(),
+                totalCollateral: 999999.Tst())
+            );
+            return host;
         }
 
         public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message)
@@ -159,6 +138,19 @@ namespace CodexReleaseTests.MarketTests
             });
         }
 
+        private ChainMonitor? SetupChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc)
+        {
+            if (!MonitorChainState) return null;
+
+            var result = new ChainMonitor(log, contracts, startUtc);
+            result.Start(() =>
+            {
+                log.Error("Failure in chain monitor. No chain updates after this point.");
+                //Assert.Fail("Failure in chain monitor.");
+            });
+            return result;
+        }
+
         private Retry GetBalanceAssertRetry()
         {
             return new Retry("AssertBalance",
@@ -168,15 +160,6 @@ namespace CodexReleaseTests.MarketTests
                 failFast: false);
         }
 
-        private Retry GetAvailabilitySpaceAssertRetry()
-        {
-            return new Retry("AssertAvailabilitySpace",
-                maxTimeout: HostBlockTTL * 3,
-                sleepAfterFail: TimeSpan.FromSeconds(10.0),
-                onFail: f => { },
-                failFast: false);
-        }
-
         private TestToken GetTstBalance(ICodexNode node)
         {
             return GetContracts().GetTestTokenBalance(node);
@@ -225,7 +208,7 @@ namespace CodexReleaseTests.MarketTests
             );
         }
 
-        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
+        public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts, string purchaseId)
         {
             var fills = GetOnChainSlotFills(possibleHosts);
             return fills.Where(f => f
@@ -233,7 +216,7 @@ namespace CodexReleaseTests.MarketTests
                 .ToArray();
         }
 
-        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
+        public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts)
         {
             var events = GetContracts().GetEvents(GetTestRunTimeRange());
             var fills = events.GetSlotFilledEvents();
@@ -245,34 +228,13 @@ namespace CodexReleaseTests.MarketTests
             }).ToArray();
         }
 
-        public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts, string purchaseId)
-        {
-            var fills = GetOnChainSlotFrees(possibleHosts);
-            return fills.Where(f => f
-                .SlotFreedEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
-                .ToArray();
-        }
-
-        public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts)
-        {
-            var events = GetContracts().GetEvents(GetTestRunTimeRange());
-            var fills = GetOnChainSlotFills(possibleHosts);
-            var frees = events.GetSlotFreedEvents();
-            return frees.Select(f =>
-            {
-                var matchingFill = fills.Single(fill => fill.SlotFilledEvent.RequestId == f.RequestId &&
-                    fill.SlotFilledEvent.SlotIndex == f.SlotIndex);
-
-                return new SlotFree(f, matchingFill.Host);
-
-            }).ToArray();
-        }
-
         protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
         {
             var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts);
 
             AssertTstBalance(client, expectedBalance, "Client balance incorrect.");
+
+            Log($"Client has paid for contract. Balance: {expectedBalance}");
         }
 
         protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
@@ -292,7 +254,9 @@ namespace CodexReleaseTests.MarketTests
 
             foreach (var pair in expectedBalances)
             {
-                AssertTstBalance(pair.Key, pair.Value, "Host was not paid for storage.");
+                AssertTstBalance(pair.Key, pair.Value, $"Host {pair.Key} was not paid for storage.");
+
+                Log($"Host {pair.Key} was paid for storage. Balance: {pair.Value}");
             }
         }
 
@@ -313,6 +277,30 @@ namespace CodexReleaseTests.MarketTests
             }
         }
 
+        protected void WaitForContractStarted(IStoragePurchaseContract r)
+        {
+            try
+            {
+                r.WaitForStorageContractStarted();
+            }
+            catch
+            {
+                // Contract failed to start. Retrieve and log every call to ReserveSlot to identify which hosts
+                // should have filled the slot.
+
+                var requestId = r.PurchaseId.ToLowerInvariant();
+                var calls = new List<ReserveSlotFunction>();
+                GetContracts().GetEvents(GetTestRunTimeRange()).GetReserveSlotCalls(calls.Add);
+
+                Log($"Request '{requestId}' failed to start. There were {calls.Count} hosts who called reserve-slot for it:");
+                foreach (var c in calls)
+                {
+                    Log($" - {c.Block.Utc} Host: {c.FromAddress} RequestId: {c.RequestId.ToHex()} SlotIndex: {c.SlotIndex}");
+                }
+                throw;
+            }
+        }
+
         private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
         {
             var fills = GetOnChainSlotFills(hosts);
@@ -332,7 +320,7 @@ namespace CodexReleaseTests.MarketTests
 
         private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract)
         {
-            return Time.Retry<DateTime>(() =>
+            return Time.Retry(() =>
             {
                 var events = GetContracts().GetEvents(GetTestRunTimeRange());
                 var submitEvent = events.GetStorageRequests().SingleOrDefault(e => e.RequestId.ToHex(false) == contract.PurchaseId);
@@ -391,6 +379,33 @@ namespace CodexReleaseTests.MarketTests
             }, description);
         }
 
+        protected TimeSpan CalculateContractFailTimespan()
+        {
+            var config = GetContracts().Deployment.Config;
+            var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
+            var periodDuration = GetPeriodDuration();
+            var gracePeriod = periodDuration;
+
+            // Each host could miss 1 proof per period,
+            // so the time we should wait is period time * requiredNum of missed proofs.
+            // Except: the proof requirement has a concept of "downtime":
+            // a segment of time where proof is not required.
+            // We calculate the probability of downtime and extend the waiting
+            // timeframe by a factor, such that all hosts are highly likely to have 
+            // failed a sufficient number of proofs.
+
+            float n = requiredNumMissedProofs;
+            return gracePeriod + periodDuration * n * GetDowntimeFactor(config);
+        }
+
+        private float GetDowntimeFactor(MarketplaceConfig config)
+        {
+            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
+            float downtime = numBlocksInDowntimeSegment;
+            float window = 256.0f;
+            var chanceOfDowntime = downtime / window;
+            return 1.0f + chanceOfDowntime + chanceOfDowntime;
+        }
         public class SlotFill
         {
             public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
@@ -403,30 +418,18 @@ namespace CodexReleaseTests.MarketTests
             public ICodexNode Host { get; }
         }
 
-        public class SlotFree
-        {
-            public SlotFree(SlotFreedEventDTO slotFreedEvent, ICodexNode host)
-            {
-                SlotFreedEvent = slotFreedEvent;
-                Host = host;
-            }
-
-            public SlotFreedEventDTO SlotFreedEvent { get; }
-            public ICodexNode Host { get; }
-        }
-
         private class MarketplaceHandle
         {
-            public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor monitor)
+            public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor? chainMonitor)
             {
                 Geth = geth;
                 Contracts = contracts;
-                Monitor = monitor;
+                ChainMonitor = chainMonitor;
             }
 
             public IGethNode Geth { get; }
             public ICodexContracts Contracts { get; }
-            public ChainMonitor Monitor { get; }
+            public ChainMonitor? ChainMonitor { get; }
         }
     }
 }

Tests/CodexReleaseTests/Utils/PurchaseParams.cs

@@ -0,0 +1,67 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.Utils
+{
+    public class PurchaseParams
+    {
+        private readonly ByteSize blockSize = 64.KB();
+
+        public PurchaseParams(int nodes, int tolerance, ByteSize uploadFilesize)
+        {
+            Nodes = nodes;
+            Tolerance = tolerance;
+            UploadFilesize = uploadFilesize;
+
+            EncodedDatasetSize = CalculateEncodedDatasetSize();
+            SlotSize = CalculateSlotSize();
+
+            Assert.That(IsPowerOfTwo(SlotSize));
+        }
+
+        public int Nodes { get; }
+        public int Tolerance { get; }
+        public ByteSize UploadFilesize { get; }
+        public ByteSize EncodedDatasetSize { get; }
+        public ByteSize SlotSize { get; }
+
+        private ByteSize CalculateSlotSize()
+        {
+            // encoded dataset is divided over the nodes.
+            // then each slot is rounded up to the nearest power-of-two blocks.
+            var numBlocks = EncodedDatasetSize.DivUp(blockSize);
+            var numSlotBlocks = 1 + ((numBlocks - 1) / Nodes); // round-up div.
+
+            // Next power of two:
+            var numSlotBlocksPow2 = NextPowerOf2(numSlotBlocks);
+            return new ByteSize(blockSize.SizeInBytes * numSlotBlocksPow2);
+        }
+
+        private ByteSize CalculateEncodedDatasetSize()
+        {
+            var numBlocks = UploadFilesize.DivUp(blockSize);
+
+            var ecK = Nodes - Tolerance;
+            var ecM = Tolerance;
+
+            // for each K blocks, we generate M parity blocks
+            var numParityBlocks = (numBlocks / ecK) * ecM;
+            var totalBlocks = numBlocks + numParityBlocks;
+
+            return new ByteSize(blockSize.SizeInBytes * totalBlocks);
+        }
+
+        private int NextPowerOf2(int n)
+        {
+            n = n - 1;
+            var lg = Convert.ToInt32(Math.Round(Math.Log2(Convert.ToDouble(n))));
+            return 1 << (lg + 1);
+        }
+
+        private static bool IsPowerOfTwo(ByteSize size)
+        {
+            var x = size.SizeInBytes;
+            return (x != 0) && ((x & (x - 1)) == 0);
+        }
+    }
+}

Tests/DistTestCore/DistTest.cs

@@ -12,70 +12,45 @@ using Assert = NUnit.Framework.Assert;
 namespace DistTestCore
 {
     [Parallelizable(ParallelScope.All)]
+    [FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
     public abstract class DistTest
     {
-        private const string TestNamespacePrefix = "cdx-";
-        private readonly Configuration configuration = new Configuration();
-        private readonly Assembly[] testAssemblies;
+        private static readonly Global global = new Global();
         private readonly FixtureLog fixtureLog;
         private readonly StatusLog statusLog;
-        private readonly object lifecycleLock = new object();
-        private readonly EntryPoint globalEntryPoint;
-        private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
-        private readonly string deployId;
-        
+        private readonly TestLifecycle lifecycle;
+        private readonly string deployId = NameUtils.MakeDeployId();
+
         public DistTest()
         {
-            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
-            testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
-            
-            deployId = NameUtils.MakeDeployId();
-
-            var logConfig = configuration.GetLogConfig();
+            var logConfig = global.Configuration.GetLogConfig();
             var startTime = DateTime.UtcNow;
             fixtureLog = FixtureLog.Create(logConfig, startTime, deployId);
             statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultK8sTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
+            fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus());
+
+            lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(startTime), global.Configuration,
+                GetWebCallTimeSet(),
+                GetK8sTimeSet(),
+                Global.TestNamespacePrefix + Guid.NewGuid().ToString(),
+                deployId,
+                ShouldWaitForCleanup()
+            );
 
             Initialize(fixtureLog);
         }
 
         [OneTimeSetUp]
-        public void GlobalSetup()
+        public static void GlobalSetup()
         {
-            fixtureLog.Log($"Starting...");
-            globalEntryPoint.Announce();
-
-            // Previous test run may have been interrupted.
-            // Begin by cleaning everything up.
-            try
-            {
-                Stopwatch.Measure(fixtureLog, "Global setup", () =>
-                {
-                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
-                });                
-            }
-            catch (Exception ex)
-            {
-                GlobalTestFailure.HasFailed = true;
-                fixtureLog.Error($"Global setup cleanup failed with: {ex}");
-                throw;
-            }
-
-            fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus());
-            fixtureLog.Log("Global setup cleanup successful");
+            global.Setup();
         }
 
         [OneTimeTearDown]
-        public void GlobalTearDown()
+        public static void GlobalTearDown()
         {
-            globalEntryPoint.Decommission(
-                // There shouldn't be any of either, but clean everything up regardless.
-                deleteKubernetesResources: true,
-                deleteTrackedFiles: true,
-                waitTillDone: true
-            );
+            global.TearDown();
         }
 
         [SetUp]
@@ -85,18 +60,6 @@ namespace DistTestCore
             {
                 Assert.Inconclusive("Skip test: Previous test failed during clean up.");
             }
-            else
-            {
-                try
-                {
-                    CreateNewTestLifecycle();
-                }
-                catch (Exception ex)
-                {
-                    fixtureLog.Error("Setup failed: " + ex);
-                    GlobalTestFailure.HasFailed = true;
-                }
-            }
         }
 
         [TearDown]
@@ -117,18 +80,18 @@ namespace DistTestCore
         {
             get
             {
-                return Get().CoreInterface;
+                return lifecycle.CoreInterface;
             }
         }
 
         public TrackedFile GenerateTestFile(ByteSize size, string label = "")
         {
-            return Get().GenerateTestFile(size, label);
+            return lifecycle.GenerateTestFile(size, label);
         }
 
         public TrackedFile GenerateTestFile(Action<IGenerateOption> options, string label = "")
         {
-            return Get().GenerateTestFile(options, label);
+            return lifecycle.GenerateTestFile(options, label);
         }
 
         /// <summary>
@@ -137,12 +100,22 @@ namespace DistTestCore
         /// </summary>
         public void ScopedTestFiles(Action action)
         {
-            Get().GetFileManager().ScopedFiles(action);
+            lifecycle.GetFileManager().ScopedFiles(action);
         }
 
         public ILog GetTestLog()
         {
-            return Get().Log;
+            return lifecycle.Log;
+        }
+
+        public IFileManager GetFileManager()
+        {
+            return lifecycle.GetFileManager();
+        }
+
+        public string GetTestNamespace()
+        {
+            return lifecycle.TestNamespace;
         }
 
         public void Log(string msg)
@@ -159,64 +132,24 @@ namespace DistTestCore
 
         public void Measure(string name, Action action)
         {
-            Stopwatch.Measure(Get().Log, name, action);
+            Stopwatch.Measure(lifecycle.Log, name, action);
         }
 
         protected TimeRange GetTestRunTimeRange()
         {
-            return new TimeRange(Get().TestStart, DateTime.UtcNow);
+            return new TimeRange(lifecycle.TestStartUtc, DateTime.UtcNow);
         }
 
         protected virtual void Initialize(FixtureLog fixtureLog)
         {
         }
 
-        protected virtual void LifecycleStart(TestLifecycle lifecycle)
-        {
-        }
-
-        protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult)
-        {
-        }
-
         protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
         {
         }
 
-        protected TestLifecycle Get()
-        {
-            lock (lifecycleLock)
-            {
-                return lifecycles[GetCurrentTestName()];
-            }
-        }
-
-        private void CreateNewTestLifecycle()
-        {
-            var testName = GetCurrentTestName();
-            fixtureLog.WriteLogTag();
-            Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
-            {
-                lock (lifecycleLock)
-                {
-                    var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(
-                        fixtureLog.CreateTestLog(),
-                        configuration,
-                        GetWebCallTimeSet(),
-                        GetK8sTimeSet(),
-                        testNamespace,
-                        deployId,
-                        ShouldWaitForCleanup());
-                    lifecycles.Add(testName, lifecycle);
-                    LifecycleStart(lifecycle);
-                }
-            });
-        }
-
         private void DisposeTestLifecycle()
         {
-            var lifecycle = Get();
             var testResult = GetTestResult();
             var testDuration = lifecycle.GetTestDuration();
             var data = lifecycle.GetPluginMetadata();
@@ -228,9 +161,7 @@ namespace DistTestCore
                 WriteEndTestLog(lifecycle.Log);
 
                 IncludeLogsOnTestFailure(lifecycle);
-                LifecycleStop(lifecycle, testResult);
                 lifecycle.DeleteAllResources();
-                lifecycles.Remove(GetCurrentTestName());
             });
         }
 
@@ -287,7 +218,7 @@ namespace DistTestCore
             var className = currentTest.ClassName;
             var methodName = currentTest.MethodName;
 
-            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
+            var testClasses = global.TestAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
             var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
 
             return testMethods.Select(m => m.GetCustomAttribute<T>())
@@ -296,19 +227,24 @@ namespace DistTestCore
                 .ToArray();
         }
 
+        protected IDownloadedLog[] DownloadAllLogs()
+        {
+            return lifecycle.DownloadAllLogs();
+        }
+
         private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
         {
             var testStatus = TestContext.CurrentContext.Result.Outcome.Status;
             if (ShouldDownloadAllLogs(testStatus))
             {
                 lifecycle.Log.Log("Downloading all container logs...");
-                lifecycle.DownloadAllLogs();
+                DownloadAllLogs();
             }
         }
 
         private bool ShouldDownloadAllLogs(TestStatus testStatus)
         {
-            if (configuration.AlwaysDownloadContainerLogs) return true;
+            if (global.Configuration.AlwaysDownloadContainerLogs) return true;
             if (!IsDownloadingLogsEnabled()) return false;
             if (testStatus == TestStatus.Failed)
             {
@@ -323,7 +259,7 @@ namespace DistTestCore
             return $"[{TestContext.CurrentContext.Test.Name}]";
         }
 
-        private DistTestResult GetTestResult()
+        public DistTestResult GetTestResult()
         {
             var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed;
             var status = TestContext.CurrentContext.Result.Outcome.Status.ToString();

Tests/DistTestCore/Global.cs

@@ -0,0 +1,65 @@
+using System.Diagnostics;
+using System.Reflection;
+using Core;
+using Logging;
+
+namespace DistTestCore
+{
+    public class Global
+    {
+        public const string TestNamespacePrefix = "cdx-";
+        public Configuration Configuration { get; } = new Configuration();
+
+        public Assembly[] TestAssemblies { get; }
+        private readonly EntryPoint globalEntryPoint;
+        private readonly ILog log;
+
+        public Global()
+        {
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            TestAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
+
+            log = new ConsoleLog();
+            globalEntryPoint = new EntryPoint(
+                log,
+                Configuration.GetK8sConfiguration(
+                    new DefaultK8sTimeSet(),
+                    TestNamespacePrefix
+                ),
+                Configuration.GetFileManagerFolder()
+            );
+        }
+
+        public void Setup()
+        {
+            try
+            {
+                Trace.Listeners.Add(new ConsoleTraceListener());
+
+                Logging.Stopwatch.Measure(log, "Global setup", () =>
+                {
+                    globalEntryPoint.Announce();
+                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
+                });
+            }
+            catch (Exception ex)
+            {
+                GlobalTestFailure.HasFailed = true;
+                log.Error($"Global setup cleanup failed with: {ex}");
+                throw;
+            }
+        }
+
+        public void TearDown()
+        {
+            globalEntryPoint.Decommission(
+                // There shouldn't be any of either, but clean everything up regardless.
+                deleteKubernetesResources: true,
+                deleteTrackedFiles: true,
+                waitTillDone: true
+            );
+
+            Trace.Flush();
+        }
+    }
+}

Tests/DistTestCore/Logs/BaseTestLog.cs

@@ -8,7 +8,7 @@ namespace DistTestCore.Logs
 
         protected BaseTestLog(ILog backingLog, string deployId)
         {
-            this.backingLog = backingLog;
+            this.backingLog = new TimestampPrefixer(backingLog);
 
             DeployId = deployId;
         }
@@ -59,24 +59,8 @@ namespace DistTestCore.Logs
 
         protected static ILog CreateMainLog(string fullName, string name)
         {
-            ILog log = new FileLog(fullName);
-            log = ApplyConsoleOutput(log);
-            return log;
-        }
-
-        private static ILog ApplyConsoleOutput(ILog log)
-        {
-            // If we're running as a release test, we'll split the log output
-            // to the console as well.
-
-            var testType = Environment.GetEnvironmentVariable("TEST_TYPE");
-            if (string.IsNullOrEmpty(testType) || testType.ToLowerInvariant() != "release-tests")
-            {
-                return log;
-            }
-
             return new LogSplitter(
-                log,
+                new FileLog(fullName),
                 new ConsoleLog()
             );
         }

Tests/DistTestCore/Logs/FixtureLog.cs

@@ -4,19 +4,14 @@ namespace DistTestCore.Logs
 {
     public class FixtureLog : BaseTestLog
     {
-        private readonly ILog backingLog;
-        private readonly string deployId;
-
         public FixtureLog(ILog backingLog, string deployId)
             : base(backingLog, deployId)
         {
-            this.backingLog = backingLog;
-            this.deployId = deployId;
         }
 
-        public TestLog CreateTestLog(string name = "")
+        public TestLog CreateTestLog(DateTime start, string name = "")
         {
-            return TestLog.Create(this, name);
+            return TestLog.Create(this, start, name);
         }
 
         public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "")

Tests/DistTestCore/Logs/TestLog.cs

@@ -1,5 +1,4 @@
 using Logging;
-using System.Xml.Linq;
 
 namespace DistTestCore.Logs
 {
@@ -11,9 +10,9 @@ namespace DistTestCore.Logs
             backingLog.Log($"*** Begin: {methodName}");
         }
 
-        public static TestLog Create(FixtureLog parentLog, string name = "")
+        public static TestLog Create(FixtureLog parentLog, DateTime start, string name = "")
         {
-            var methodName = NameUtils.GetTestMethodName(name);
+            var methodName = NameUtils.GetTestLogFileName(start, name);
             var fullName = Path.Combine(parentLog.GetFullName(), methodName);
             var backingLog = CreateMainLog(fullName, name);
             return new TestLog(backingLog, methodName, parentLog.DeployId);

Tests/DistTestCore/NameUtils.cs

@@ -5,6 +5,11 @@ namespace DistTestCore
 {
     public static class NameUtils
     {
+        public static string GetTestLogFileName(DateTime start, string name = "")
+        {
+            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{GetTestMethodName(name)}";
+        }
+
         public static string GetTestMethodName(string name = "")
         {
             if (!string.IsNullOrEmpty(name)) return name;
@@ -16,7 +21,7 @@ namespace DistTestCore
         public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
         {
             var folder = DetermineFolder(config, start);
-            var fixtureName = GetFixtureName(name, start);
+            var fixtureName = GetRawFixtureName();
             return Path.Combine(folder, fixtureName);
         }
 
@@ -85,13 +90,6 @@ namespace DistTestCore
                Pad(start.Day));
         }
 
-        private static string GetFixtureName(string name, DateTime start)
-        {
-            var fixtureName = GetRawFixtureName();
-            if (!string.IsNullOrEmpty(name)) fixtureName = name;
-            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{fixtureName.Replace('.', '-')}";
-        }
-
         private static string Pad(int n)
         {
             return n.ToString().PadLeft(2, '0');

Tests/DistTestCore/TestLifecycle.cs

@@ -26,7 +26,7 @@ namespace DistTestCore
             WebCallTimeSet = webCallTimeSet;
             K8STimeSet = k8sTimeSet;
             TestNamespace = testNamespace;
-            TestStart = DateTime.UtcNow;
+            TestStartUtc = DateTime.UtcNow;
 
             entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(k8sTimeSet, this, testNamespace), configuration.GetFileManagerFolder(), webCallTimeSet, k8sTimeSet);
             metadata = entryPoint.GetPluginMetadata();
@@ -36,7 +36,7 @@ namespace DistTestCore
             log.WriteLogTag();
         }
 
-        public DateTime TestStart { get; }
+        public DateTime TestStartUtc { get; }
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public IWebCallTimeSet WebCallTimeSet { get; }
@@ -76,7 +76,7 @@ namespace DistTestCore
 
         public TimeSpan GetTestDuration()
         {
-            return DateTime.UtcNow - TestStart;
+            return DateTime.UtcNow - TestStartUtc;
         }
 
         public void OnContainersStarted(RunningPod rc)

Tests/ExperimentalTests/AutoBootstrapDistTest.cs

@@ -1,47 +1,35 @@
 using CodexClient;
 using CodexPlugin;
-using DistTestCore;
 using NUnit.Framework;
 
 namespace CodexTests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {
-        private readonly Dictionary<TestLifecycle, ICodexNode> bootstrapNodes = new Dictionary<TestLifecycle, ICodexNode>();
+        private bool isBooting = false;
+
+        public ICodexNode BootstrapNode { get; private set; } = null!;
 
         [SetUp]
-        public void SetUpBootstrapNode()
+        public void SetupBootstrapNode()
         {
-            var tl = Get();
-            if (!bootstrapNodes.ContainsKey(tl))
-            {
-                bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace)));
-            }
+            isBooting = true;
+            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP_" + GetTestNamespace()));
+            isBooting = false;
         }
 
         [TearDown]
         public void TearDownBootstrapNode()
         {
-            bootstrapNodes.Remove(Get());
+            BootstrapNode.Stop(waitTillStopped: false);
         }
 
         protected override void OnCodexSetup(ICodexSetup setup)
         {
+            if (isBooting) return;
+
             var node = BootstrapNode;
             if (node != null) setup.WithBootstrapNode(node);
         }
-
-        protected ICodexNode? BootstrapNode
-        {
-            get
-            {
-                var tl = Get();
-                if (bootstrapNodes.TryGetValue(tl, out var node))
-                {
-                    return node;
-                }
-                return null;
-            }
-        }
     }
 }

Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs

@@ -31,7 +31,7 @@ namespace ExperimentalTests.BasicTests
             );
 
             var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
+            var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
 
             var numberOfHosts = 5;
             var hosts = StartCodex(numberOfHosts, s => s
@@ -48,7 +48,7 @@ namespace ExperimentalTests.BasicTests
 
             foreach (var host in hosts)
             {
-                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
+                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance), "Host initial balance");
 
                 var availability = new CreateStorageAvailability(
                     totalSpace: 10.GB(),
@@ -66,7 +66,7 @@ namespace ExperimentalTests.BasicTests
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), clientInitialBalance)));
 
-            AssertBalance(contracts, client, Is.EqualTo(clientInitialBalance));
+            AssertBalance(contracts, client, Is.EqualTo(clientInitialBalance), "Client initial balance");
 
             var uploadCid = client.UploadFile(testFile);
 

Tests/ExperimentalTests/CodexDistTest.cs

@@ -1,6 +1,5 @@
 using BlockchainUtils;
 using CodexClient;
-using CodexClient.Hooks;
 using CodexContractsPlugin;
 using CodexNetDeployer;
 using CodexPlugin;
@@ -17,85 +16,14 @@ using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using OverwatchTranscript;
-using Utils;
 
 namespace CodexTests
 {
-    public class CodexLogTrackerProvider  : ICodexHooksProvider
-    {
-        private readonly Action<ICodexNode> addNode;
-
-        public CodexLogTrackerProvider(Action<ICodexNode> addNode)
-        {
-            this.addNode = addNode;
-        }
-
-        // See TestLifecycle.cs DownloadAllLogs()
-        public ICodexNodeHooks CreateHooks(string nodeName)
-        {
-            return new CodexLogTracker(addNode);
-        }
-
-        public class CodexLogTracker : ICodexNodeHooks
-        {
-            private readonly Action<ICodexNode> addNode;
-
-            public CodexLogTracker(Action<ICodexNode> addNode)
-            {
-                this.addNode = addNode;
-            }
-
-            public void OnFileDownloaded(ByteSize size, ContentId cid)
-            {
-            }
-
-            public void OnFileDownloading(ContentId cid)
-            {
-            }
-
-            public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
-            {
-            }
-
-            public void OnFileUploading(string uid, ByteSize size)
-            {
-            }
-
-            public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
-            {
-                addNode(node);
-            }
-
-            public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
-            {
-            }
-
-            public void OnNodeStopping()
-            {
-            }
-
-            public void OnStorageAvailabilityCreated(StorageAvailability response)
-            {
-            }
-
-            public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
-            {
-            }
-
-            public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
-            {
-            }
-        }
-    }
-
     public class CodexDistTest : DistTest
     {
-        private static readonly object _lock = new object();
-        private static readonly Dictionary<TestLifecycle, CodexTranscriptWriter> writers = new Dictionary<TestLifecycle, CodexTranscriptWriter>();
-        private static readonly Dictionary<TestLifecycle, BlockCache> blockCaches = new Dictionary<TestLifecycle, BlockCache>();
-
-        // this entire structure is not good and needs to be destroyed at the earliest convenience:
-        private static readonly Dictionary<TestLifecycle, List<ICodexNode>> nodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
+        private readonly BlockCache blockCache = new BlockCache();
+        private readonly List<ICodexNode> nodes = new List<ICodexNode>();
+        private CodexTranscriptWriter? writer;
 
         public CodexDistTest()
         {
@@ -105,41 +33,25 @@ namespace CodexTests
             ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
         }
 
+        [SetUp]
+        public void SetupCodexDistTest()
+        {
+            writer = SetupTranscript();
+        }
+
+        [TearDown]
+        public void TearDownCodexDistTest()
+        {
+            TeardownTranscript();
+        }
+
         protected override void Initialize(FixtureLog fixtureLog)
         {
             var localBuilder = new LocalCodexBuilder(fixtureLog);
             localBuilder.Intialize();
             localBuilder.Build();
-        }
 
-        protected override void LifecycleStart(TestLifecycle lifecycle)
-        {
-            base.LifecycleStart(lifecycle);
-            SetupTranscript(lifecycle);
-
-            Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n =>
-            {
-                lock (_lock)
-                {
-                    if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List<ICodexNode>());
-                    nodes[lifecycle].Add(n);
-                }
-            }));
-        }
-
-        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
-        {
-            base.LifecycleStop(lifecycle, result);
-            TeardownTranscript(lifecycle, result);
-
-            if (!result.Success)
-            {
-                lock (_lock)
-                {
-                    var codexNodes = nodes[lifecycle];
-                    foreach (var node in codexNodes) node.DownloadLog();
-                }
-            }
+            Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(nodes.Add));
         }
 
         public ICodexNode StartCodex()
@@ -170,7 +82,7 @@ namespace CodexTests
 
         public IGethNode StartGethNode(Action<IGethSetup> setup)
         {
-            return Ci.StartGethNode(GetBlockCache(), setup);
+            return Ci.StartGethNode(blockCache, setup);
         }
 
         public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
@@ -180,10 +92,10 @@ namespace CodexTests
 
         public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
         {
-            return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager());
+            return new PeerDownloadTestHelpers(GetTestLog(), GetFileManager());
         }
 
-        public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
+        public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg)
         {
             Assert.Fail("Depricated, use MarketplaceAutobootstrapDistTest assertBalances instead.");
             AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
@@ -259,82 +171,47 @@ namespace CodexTests
             return null;
         }
 
-        private void SetupTranscript(TestLifecycle lifecycle)
+        private CodexTranscriptWriter? SetupTranscript()
         {
             var attr = GetTranscriptAttributeOfCurrentTest();
-            if (attr == null) return;
+            if (attr == null) return null;
 
             var config = new CodexTranscriptWriterConfig(
+                attr.OutputFilename,
                 attr.IncludeBlockReceivedEvents
             );
 
-            var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
+            var log = new LogPrefixer(GetTestLog(), "(Transcript) ");
             var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log));
             Ci.AddCodexHooksProvider(writer);
-            lock (_lock)
-            {
-                writers.Add(lifecycle, writer);
-            }
+            return writer;
         }
 
-        private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result)
+        private void TeardownTranscript()
         {
-            var attr = GetTranscriptAttributeOfCurrentTest();
-            if (attr == null) return;
-
-            var outputFilepath = GetOutputFullPath(lifecycle, attr);
-
-            CodexTranscriptWriter writer = null!;
-            lock (_lock)
-            {
-                writer = writers[lifecycle];
-                writers.Remove(lifecycle);
-            }
+            if (writer == null) return;
 
+            var result = GetTestResult();
+            var log = GetTestLog();
             writer.AddResult(result.Success, result.Result);
-
             try
             {
-                Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () =>
+                Stopwatch.Measure(log, "Transcript.ProcessLogs", () =>
                 {
-                    writer.ProcessLogs(lifecycle.DownloadAllLogs());
+                    writer.ProcessLogs(DownloadAllLogs());
                 });
 
-                Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () =>
+                Stopwatch.Measure(log, $"Transcript.FinalizeWriter", () =>
                 {
-                    writer.IncludeFile(lifecycle.Log.GetFullName());
-                    writer.Finalize(outputFilepath);
+                    writer.IncludeFile(log.GetFullName() + ".log");
+                    writer.FinalizeWriter();
                 });
             }
             catch (Exception ex)
             {
-                lifecycle.Log.Error("Failure during transcript teardown: " + ex);
+                log.Error("Failure during transcript teardown: " + ex);
             }
         }
-
-        private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr)
-        {
-            var outputPath = Path.GetDirectoryName(lifecycle.Log.GetFullName());
-            if (outputPath == null) throw new Exception("Logfile path is null");
-            var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.GetFullName());
-            if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty");
-            var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename);
-            if (!outputFile.EndsWith(".owts")) outputFile += ".owts";
-            return outputFile;
-        }
-
-        private BlockCache GetBlockCache()
-        {
-            var lifecycle = Get();
-            lock (_lock)
-            {
-                if (!blockCaches.ContainsKey(lifecycle))
-                {
-                    blockCaches[lifecycle] = new BlockCache();
-                }
-            }
-            return blockCaches[lifecycle];
-        }
     }
 
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]

Tests/ExperimentalTests/CodexLogTrackerProvider.cs

@@ -0,0 +1,73 @@
+using CodexClient;
+using CodexClient.Hooks;
+using Utils;
+
+namespace CodexTests
+{
+    public class CodexLogTrackerProvider  : ICodexHooksProvider
+    {
+        private readonly Action<ICodexNode> addNode;
+
+        public CodexLogTrackerProvider(Action<ICodexNode> addNode)
+        {
+            this.addNode = addNode;
+        }
+
+        // See TestLifecycle.cs DownloadAllLogs()
+        public ICodexNodeHooks CreateHooks(string nodeName)
+        {
+            return new CodexLogTracker(addNode);
+        }
+
+        public class CodexLogTracker : ICodexNodeHooks
+        {
+            private readonly Action<ICodexNode> addNode;
+
+            public CodexLogTracker(Action<ICodexNode> addNode)
+            {
+                this.addNode = addNode;
+            }
+
+            public void OnFileDownloaded(ByteSize size, ContentId cid)
+            {
+            }
+
+            public void OnFileDownloading(ContentId cid)
+            {
+            }
+
+            public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+            {
+            }
+
+            public void OnFileUploading(string uid, ByteSize size)
+            {
+            }
+
+            public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+            {
+                addNode(node);
+            }
+
+            public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+            {
+            }
+
+            public void OnNodeStopping()
+            {
+            }
+
+            public void OnStorageAvailabilityCreated(StorageAvailability response)
+            {
+            }
+
+            public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+            {
+            }
+
+            public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+            {
+            }
+        }
+    }
+}

Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -21,7 +21,7 @@ namespace ExperimentalTests.DownloadConnectivityTests
         public void MarketplaceDoesNotInterfereWithPeerDownload()
         {
             var geth = StartGethNode(s => s.IsMiner());
-            var contracts = Ci.StartCodexContracts(geth);
+            var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
             var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TstWei())));
 

Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -31,7 +31,7 @@ namespace ExperimentalTests.PeerDiscoveryTests
         public void MarketplaceDoesNotInterfereWithPeerDiscovery()
         {
             var geth = StartGethNode(s => s.IsMiner());
-            var contracts = Ci.StartCodexContracts(geth);
+            var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
             var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TstWei())));
 

Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs

@@ -0,0 +1,31 @@
+using GethPlugin;
+using NUnit.Framework;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EthAccountEqualityTests
+    {
+        [Test]
+        public void Accounts()
+        {
+            var account1 = EthAccountGenerator.GenerateNew();
+            var account2 = EthAccountGenerator.GenerateNew();
+
+            Assert.That(account1, Is.EqualTo(account1));
+            Assert.That(account1 == account1);
+            Assert.That(account1 != account2);
+        }
+
+        [Test]
+        public void Addresses()
+        {
+            var address1 = EthAccountGenerator.GenerateNew().EthAddress;
+            var address2 = EthAccountGenerator.GenerateNew().EthAddress;
+
+            Assert.That(address1, Is.EqualTo(address1));
+            Assert.That(address1 == address1);
+            Assert.That(address1 != address2);
+        }
+    }
+}

Tools/AutoClient/CodexWrapper.cs

@@ -7,6 +7,7 @@ namespace AutoClient
     public class CodexWrapper
     {
         private readonly App app;
+        private static readonly Random r = new Random();
 
         public CodexWrapper(App app, ICodexNode node)
         {
@@ -26,11 +27,11 @@ namespace AutoClient
             var result = Node.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 CollateralPerByte = app.Config.CollateralPerByte.TstWei(),
-                Duration = TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
+                Duration = GetDuration(),
                 Expiry = TimeSpan.FromMinutes(app.Config.ContractExpiryMinutes),
                 MinRequiredNumberOfNodes = Convert.ToUInt32(app.Config.NumHosts),
                 NodeFailureTolerance = Convert.ToUInt32(app.Config.HostTolerance),
-                PricePerBytePerSecond = app.Config.PricePerBytePerSecond.TstWei(),
+                PricePerBytePerSecond = GetPricePerBytePerSecond(),
                 ProofProbability = 15
             });
             return result;
@@ -40,5 +41,25 @@ namespace AutoClient
         {
             return Node.GetPurchaseStatus(pid);
         }
+
+        private TestToken GetPricePerBytePerSecond()
+        {
+            var i = app.Config.PricePerBytePerSecond;
+            i -= 100;
+            i += r.Next(0, 1000);
+
+            return i.TstWei();
+        }
+
+        private TimeSpan GetDuration()
+        {
+            var i = app.Config.ContractDurationMinutes;
+            var day = 60 * 24;
+            i -= day;
+            i -= 10; // We don't want to accidentally hit exactly 7 days because that's the limit of the storage node availabilities.
+            i += r.Next(0, day * 2);
+
+            return TimeSpan.FromMinutes(i);
+        }
     }
 }

Tools/AutoClient/Configuration.cs

@@ -59,6 +59,9 @@ namespace AutoClient
             "/root/codex-testnet-starter/scripts/eth_7.address" + ";" +
             "/root/codex-testnet-starter/scripts/eth_8.address";
 
+        [Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")]
+        public int SlowModeDelayMinutes { get; set; } = 60 * 1;
+
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileSaver.cs

@@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore
     public interface IFileSaverEventHandler
     {
         void SaveChanges();
+    }
+
+    public interface IFileSaverResultHandler
+    {
+        void OnSuccess();
         void OnFailure();
     }
 
@@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore
         private readonly Stats stats;
         private readonly string folderFile;
         private readonly FileStatus entry;
-        private readonly IFileSaverEventHandler handler;
+        private readonly IFileSaverEventHandler saveHandler;
+        private readonly IFileSaverResultHandler resultHandler;
 
-        public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
+        public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
         {
             this.log = log;
             this.loadBalancer = loadBalancer;
             this.stats = stats;
             this.folderFile = folderFile;
             this.entry = entry;
-            this.handler = handler;
+            this.saveHandler = saveHandler;
+            this.resultHandler = resultHandler;
         }
 
         public void Process()
@@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore
             loadBalancer.DispatchOnCodex(instance =>
             {
                 entry.CodexNodeId = instance.Node.GetName();
-                handler.SaveChanges();
+                saveHandler.SaveChanges();
 
-                var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
+                var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
                 run.Process();
             });
         }
@@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore
         {
             loadBalancer.DispatchOnSpecificCodex(instance =>
             {
-                var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
+                var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
                 run.Process();
             }, entry.CodexNodeId);
         }
@@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore
         private readonly Stats stats;
         private readonly string folderFile;
         private readonly FileStatus entry;
-        private readonly IFileSaverEventHandler handler;
+        private readonly IFileSaverEventHandler saveHandler;
+        private readonly IFileSaverResultHandler resultHandler;
         private readonly QuotaCheck quotaCheck;
 
-        public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
+        public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
         {
             this.log = log;
             this.instance = instance;
             this.stats = stats;
             this.folderFile = folderFile;
             this.entry = entry;
-            this.handler = handler;
+            this.saveHandler = saveHandler;
+            this.resultHandler = resultHandler;
             quotaCheck = new QuotaCheck(log, folderFile, instance);
         }
 
@@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore
                 Thread.Sleep(TimeSpan.FromMinutes(1.0));
             }
             Log("Could not upload: Insufficient local storage quota.");
-            handler.OnFailure();
+            resultHandler.OnFailure();
             return false;
         }
 
@@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore
                 entry.BasicCid = string.Empty;
                 stats.FailedUploads++;
                 log.Error("Failed to upload: " + exc);
-                handler.OnFailure();
+                resultHandler.OnFailure();
             }
-            handler.SaveChanges();
+            saveHandler.SaveChanges();
         }
 
         private void CreateNewPurchase()
@@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore
                 WaitForStarted(request);
 
                 stats.StorageRequestStats.SuccessfullyStarted++;
-                handler.SaveChanges();
+                saveHandler.SaveChanges();
 
                 Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}");
+                resultHandler.OnSuccess();
             }
             catch (Exception exc)
             {
-                entry.ClearPurchase();
-                handler.SaveChanges();
-
+                entry.EncodedCid = string.Empty;
+                entry.PurchaseId = string.Empty;
+                saveHandler.SaveChanges();
                 log.Error("Failed to start new purchase: " + exc);
-                handler.OnFailure();
+                resultHandler.OnFailure();
             }
         }
 
@@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore
                     throw new Exception("CID received from storage request was not protected.");
                 }
 
-                handler.SaveChanges();
+                saveHandler.SaveChanges();
                 Log("Saved new purchaseId: " + entry.PurchaseId);
                 return request;
             }
@@ -287,9 +297,9 @@ namespace AutoClient.Modes.FolderStore
                         else if (!update.IsSubmitted)
                         {
                             Log("Request failed to start. State: " + update.State);
-            
-                            entry.ClearPurchase();
-                            handler.SaveChanges();
+                            entry.EncodedCid = string.Empty;
+                            entry.PurchaseId = string.Empty;
+                            saveHandler.SaveChanges();
                             return;
                         }
                     }
@@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore
             }
             catch (Exception exc)
             {
-                handler.OnFailure();
+                resultHandler.OnFailure();
                 Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}");
                 throw;
             }

Tools/AutoClient/Modes/FolderStore/FolderSaver.cs

@@ -11,14 +11,16 @@ namespace AutoClient.Modes.FolderStore
         private readonly JsonFile<FolderStatus> statusFile;
         private readonly FolderStatus status;
         private readonly BalanceChecker balanceChecker;
+        private readonly SlowModeHandler slowModeHandler;
         private int changeCounter = 0;
-        private int failureCount = 0;
+        private int saveFolderJsonCounter = 0;
 
         public FolderSaver(App app, LoadBalancer loadBalancer)
         {
             this.app = app;
             this.loadBalancer = loadBalancer;
             balanceChecker = new BalanceChecker(app);
+            slowModeHandler = new SlowModeHandler(app);
 
             statusFile = new JsonFile<FolderStatus>(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename));
             status = statusFile.Load();
@@ -26,10 +28,11 @@ namespace AutoClient.Modes.FolderStore
 
         public void Run()
         {
+            saveFolderJsonCounter = 0;
+
             var folderFiles = Directory.GetFiles(app.Config.FolderToStore);
             if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore);
 
-            var saveFolderJsonCounter = 0;
             balanceChecker.Check();
             foreach (var folderFile in folderFiles)
             {
@@ -41,35 +44,30 @@ namespace AutoClient.Modes.FolderStore
                     SaveFile(folderFile);
                 }
 
-                if (failureCount > 3)
-                {
-                    app.Log.Error("Failure count reached threshold. Stopping...");
-                    app.Cts.Cancel();
-                    return;
-                }
-
-                if (changeCounter > 1)
-                {
-                    changeCounter = 0;
-                    saveFolderJsonCounter++;
-                    if (saveFolderJsonCounter > 5)
-                    {
-                        saveFolderJsonCounter = 0;
-                        if (failureCount > 0)
-                        {
-                            app.Log.Log($"Failure count is reset. (Was: {failureCount})");
-                            failureCount = 0;
-                        }
-                        balanceChecker.Check();
-                        SaveFolderSaverJsonFile();
-                    }
-                }
+                slowModeHandler.Check();
+                
+                CheckAndSaveChanges();
 
                 statusFile.Save(status);
                 Thread.Sleep(100);
             }
         }
 
+        private void CheckAndSaveChanges()
+        {
+            if (changeCounter > 1)
+            {
+                changeCounter = 0;
+                saveFolderJsonCounter++;
+                if (saveFolderJsonCounter > 5)
+                {
+                    saveFolderJsonCounter = 0;
+                    balanceChecker.Check();
+                    SaveFolderSaverJsonFile();
+                }
+            }
+        }
+
         private void SaveFile(string folderFile)
         {
             var localFilename = Path.GetFileName(folderFile);
@@ -114,7 +112,6 @@ namespace AutoClient.Modes.FolderStore
         }
 
         private const int MinCodexStorageFilesize = 262144;
-        private readonly Random random = new Random();
         private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " +
             $"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " +
             $"file is larger than the minimal size. The following is pseudo-random: ";
@@ -135,7 +132,7 @@ namespace AutoClient.Modes.FolderStore
         {
             var fixedLength = entry.Filename.PadRight(35);
             var prefix = $"[{fixedLength}] ";
-            return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this);
+            return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler);
         }
 
         public void SaveChanges()
@@ -143,10 +140,5 @@ namespace AutoClient.Modes.FolderStore
             statusFile.Save(status);
             changeCounter++;
         }
-
-        public void OnFailure()
-        {
-            failureCount++;
-        }
     }
 }

Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs

@@ -0,0 +1,54 @@
+namespace AutoClient.Modes.FolderStore
+{
+    public class SlowModeHandler : IFileSaverResultHandler
+    {
+        private readonly App app;
+        private int failureCount = 0;
+        private bool slowMode = false;
+        private int recoveryCount = 0;
+
+        public SlowModeHandler(App app)
+        {
+            this.app = app;
+        }
+
+        public void OnSuccess()
+        {
+            failureCount = 0;
+            if (slowMode)
+            {
+                recoveryCount++;
+                if (recoveryCount > 3)
+                {
+                    Log("Recovery limit reached. Exiting slow mode.");
+                    slowMode = false;
+                    failureCount = 0;
+                }
+            }
+        }
+
+        public void OnFailure()
+        {
+            failureCount++;
+            if (failureCount > 3 && !slowMode)
+            {
+                Log("Failure limit reached. Entering slow mode.");
+                slowMode = true;
+                recoveryCount = 0;
+            }
+        }
+
+        public void Check()
+        {
+            if (slowMode)
+            {
+                Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes));
+            }
+        }
+
+        private void Log(string msg)
+        {
+            app.Log.Log(msg);
+        }
+    }
+}

Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs

@@ -8,7 +8,7 @@ namespace BiblioTech.CodexChecking
     public interface ICheckResponseHandler
     {
         Task CheckNotStarted();
-        Task NowCompleted(ulong userId, string checkName);
+        Task NowCompleted(string checkName);
         Task GiveRoleReward();
         
         Task InvalidData();
@@ -37,7 +37,7 @@ namespace BiblioTech.CodexChecking
         public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId)
         {
             var check = repo.GetOrCreate(userId).DownloadCheck;
-            if (string.IsNullOrEmpty(check.UniqueData))
+            if (IsUniqueDataStale(check))
             {
                 check.UniqueData = GenerateUniqueData();
                 repo.SaveChanges();
@@ -69,7 +69,7 @@ namespace BiblioTech.CodexChecking
         public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId)
         {
             var check = repo.GetOrCreate(userId).UploadCheck;
-            if (string.IsNullOrEmpty(check.UniqueData))
+            if (IsUniqueDataStale(check))
             {
                 check.UniqueData = GenerateUniqueData();
                 repo.SaveChanges();
@@ -111,6 +111,15 @@ namespace BiblioTech.CodexChecking
             return $"{RandomBusyMessage.Get().Substring(5)}{RandomUtils.GenerateRandomString(12)}";
         }
 
+        private bool IsUniqueDataStale(TransferCheck check)
+        {
+            var expiry = DateTime.UtcNow - TimeSpan.FromMinutes(10.0);
+
+            return
+                string.IsNullOrEmpty(check.UniqueData) ||
+                check.CompletedUtc < expiry;
+        }
+
         private string UploadData(string uniqueData)
         {
             var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString());
@@ -192,7 +201,7 @@ namespace BiblioTech.CodexChecking
 
         private async Task CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId, string checkName)
         {
-            await handler.NowCompleted(userId, checkName);
+            await handler.NowCompleted(checkName);
 
             check.CompletedUtc = DateTime.UtcNow;
             repo.SaveChanges();

Tools/BiblioTech/Commands/CheckResponseHandler.cs

@@ -74,10 +74,24 @@ namespace BiblioTech.Commands
             await context.Followup("The received data didn't match. Check has failed.");
         }
 
-        public async Task NowCompleted(ulong userId, string checkName)
+        public async Task NowCompleted(string checkName)
         {
-            await context.Followup("Successfully completed the check!");
-            await Program.AdminChecker.SendInAdminChannel($"User <@{userId}> has completed check: {checkName}");
+            // check if eth address is known for user.
+            var data = Program.UserRepo.GetUser(user);
+            if (data.CurrentAddress == null)
+            {
+                await context.Followup($"Successfully completed the check!{Environment.NewLine}" +
+                    $"You haven't yet set your ethereum address. Consider using '/set' to set it.{Environment.NewLine}" +
+                    $"(You can find your address in the 'eth.address' file of your Codex node.)");
+
+                await Program.AdminChecker.SendInAdminChannel($"User <@{user.Id}> has completed check: {checkName}" +
+                    $" - EthAddress not set for user. User was reminded.");
+            }
+            else
+            {
+                await context.Followup("Successfully completed the check!");
+                await Program.AdminChecker.SendInAdminChannel($"User <@{user.Id}> has completed check: {checkName}");
+            }
         }
 
         public async Task ToAdminChannel(string msg)

Tools/CodexNetDeployer/Deployer.cs

@@ -64,8 +64,12 @@ namespace CodexNetDeployer
             var gethDeployment = DeployGeth(ci);
             var gethNode = ci.WrapGethDeployment(gethDeployment, new BlockCache());
 
+            var bootNode = ci.StartCodexNode();
+            var versionInfo = bootNode.GetDebugInfo().Version;
+            bootNode.Stop(waitTillStopped: true);
+
             Log("Geth started. Deploying Codex contracts...");
-            var contractsDeployment = ci.DeployCodexContracts(gethNode);
+            var contractsDeployment = ci.DeployCodexContracts(gethNode, versionInfo);
             var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
             Log("Codex contracts deployed.");
 

Tools/TestNetRewarder/Configuration.cs

@@ -33,6 +33,9 @@ namespace TestNetRewarder
         [Uniform("proof-submitted-events", "pse", "PROOFSUBMITTEDEVENTS", false, "When greater than zero, chain event summary will include proof-submitted events.")]
         public int ShowProofSubmittedEvents { get; set; } = 0; // Defaulted to zero, aprox 7 to 10 such events every 2 minutes in testnet (from autoclient alone!)
 
+        [Uniform("proof-period-report-hours", "pprh", "PROOFPERIODREPORTHOURS", false, "Frequency in hours with which proof period reports are created.")]
+        public int ProofReportHours { get; set; } = 24;
+
         public string LogPath
         {
             get

Tools/TestNetRewarder/Processor.cs

@@ -22,6 +22,8 @@ namespace TestNetRewarder
             this.log = log;
             lastPeriodUpdateUtc = DateTime.UtcNow;
 
+            if (config.ProofReportHours < 1) throw new Exception("ProofReportHours must be one or greater");
+
             builder = new RequestBuilder();
             eventsFormatter = new EventsFormatter(config);
 
@@ -79,7 +81,7 @@ namespace TestNetRewarder
         private void ProcessPeriodUpdate()
         {
             if (config.ShowProofPeriodReports < 1) return;
-            if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(1.0))) return;
+            if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(config.ProofReportHours))) return;
             lastPeriodUpdateUtc = DateTime.UtcNow;
 
             eventsFormatter.ProcessPeriodReports(chainState.PeriodMonitor.GetAndClearReports());

Tools/TraceContract/ChainRequestTracker.cs

@@ -0,0 +1,103 @@
+using System.Numerics;
+using BlockchainUtils;
+using CodexContractsPlugin.ChainMonitor;
+using Utils;
+
+namespace TraceContract
+{
+    public class ChainRequestTracker : IChainStateChangeHandler
+    {
+        private readonly string requestId;
+        private readonly Output output;
+
+        public ChainRequestTracker(Output output, string requestId)
+        {
+            this.requestId = requestId.ToLowerInvariant();
+            this.output = output;
+        }
+
+        public bool IsFinished { get; private set; } = false;
+        public DateTime FinishUtc { get; private set; } = DateTime.MinValue;
+
+        public void OnError(string msg)
+        {
+        }
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent)) output.LogRequestCreated(requestEvent);
+        }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestCancelled(requestEvent);
+            }
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestFailed(requestEvent);
+            }
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestFinished(requestEvent);
+            }
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogRequestStarted(requestEvent);
+            }
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotFilled(requestEvent, host, slotIndex);
+            }
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotFreed(requestEvent, slotIndex);
+            }
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotReservationsFull(requestEvent, slotIndex);
+            }
+        }
+
+        private bool IsMyRequest(RequestEvent requestEvent)
+        {
+            return requestId == requestEvent.Request.Request.Id.ToLowerInvariant();
+        }
+    }
+
+}

Tools/TraceContract/ChainTracer.cs

@@ -0,0 +1,125 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
+using Logging;
+using Nethereum.Hex.HexConvertors.Extensions;
+using Utils;
+
+namespace TraceContract
+{
+    public class ChainTracer
+    {
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly Input input;
+        private readonly Output output;
+
+        public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output)
+        {
+            this.log = log;
+            this.contracts = contracts;
+            this.input = input;
+            this.output = output;
+        }
+
+        public TimeRange TraceChainTimeline()
+        {
+            log.Log("Querying blockchain...");
+            var request = GetRequest();
+            if (request == null) throw new Exception("Failed to find the purchase in the last week of transactions.");
+
+            log.Log($"Request started at {request.Block.Utc}");
+            var contractEnd = RunToContractEnd(request);
+
+            var requestTimeline = new TimeRange(request.Block.Utc.AddMinutes(-1.0), contractEnd.AddMinutes(1.0));
+            log.Log($"Request timeline: {requestTimeline.From} -> {requestTimeline.To}");
+
+            // For this timeline, we log all the calls to reserve-slot.
+            var events = contracts.GetEvents(requestTimeline);
+
+            events.GetReserveSlotCalls(call =>
+            {
+                if (IsThisRequest(call.RequestId))
+                {
+                    output.LogReserveSlotCall(call);
+                    log.Log("Found reserve-slot call for slotIndex " + call.SlotIndex);
+                }
+            });
+
+            log.Log("Writing blockchain output...");
+            output.WriteContractEvents();
+
+            return requestTimeline;
+        }
+
+        private DateTime RunToContractEnd(Request request)
+        {
+            var utc = request.Block.Utc.AddMinutes(-1.0);
+            var tracker = new ChainRequestTracker(output, input.PurchaseId);
+            var ignoreLog = new NullLog();
+            var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false);
+
+            var atNow = false;
+            while (!tracker.IsFinished && !atNow)
+            {
+                utc += TimeSpan.FromHours(1.0);
+                if (utc > DateTime.UtcNow)
+                {
+                    log.Log("Caught up to present moment without finding contract end.");
+                    utc = DateTime.UtcNow;
+                    atNow = true;
+                }
+
+                log.Log($"Querying up to {utc}");
+                chainState.Update(utc);
+            }
+
+            if (atNow) return utc;
+            return tracker.FinishUtc;
+        }
+
+        private Request? GetRequest()
+        {
+            var request = FindRequest(LastHour());
+            if (request == null) request = FindRequest(LastDay());
+            if (request == null) request = FindRequest(LastWeek());
+            return request;
+        }
+
+        private Request? FindRequest(TimeRange timeRange)
+        {
+            var events = contracts.GetEvents(timeRange);
+            var requests = events.GetStorageRequests();
+
+            foreach (var r in requests)
+            {
+                if (IsThisRequest(r.RequestId))
+                {
+                    return r;
+                }
+            }
+
+            return null;
+        }
+
+        private bool IsThisRequest(byte[] requestId)
+        {
+            return requestId.ToHex().ToLowerInvariant() == input.PurchaseId.ToLowerInvariant();
+        }
+
+        private static TimeRange LastHour()
+        {
+            return new TimeRange(DateTime.UtcNow.AddHours(-1.0), DateTime.UtcNow);
+        }
+
+        private static TimeRange LastDay()
+        {
+            return new TimeRange(DateTime.UtcNow.AddDays(-1.0), DateTime.UtcNow);
+        }
+
+        private static TimeRange LastWeek()
+        {
+            return new TimeRange(DateTime.UtcNow.AddDays(-7.0), DateTime.UtcNow);
+        }
+    }
+}

Tools/TraceContract/ElasticSearchLogDownloader.cs

@@ -0,0 +1,253 @@
+using System.Text;
+using Core;
+using Logging;
+using Utils;
+using WebUtils;
+
+namespace TraceContract
+{
+    public class ElasticSearchLogDownloader
+    {
+        private readonly ILog log;
+        private readonly IPluginTools tools;
+        private readonly Config config;
+
+        public ElasticSearchLogDownloader(ILog log, IPluginTools tools, Config config)
+        {
+            this.log = log;
+            this.tools = tools;
+            this.config = config;
+        }
+
+        public void Download(LogFile targetFile, string podName, DateTime startUtc, DateTime endUtc)
+        {
+            try
+            {
+                DownloadLog(targetFile, podName, startUtc, endUtc);
+            }
+            catch (Exception ex)
+            {
+                log.Error("Failed to download log: " + ex);
+            }
+        }
+
+        private void DownloadLog(LogFile targetFile, string podName, DateTime startUtc, DateTime endUtc)
+        {
+            log.Log($"Downloading log (from ElasticSearch) for pod '{podName}' within time range: " +
+                $"{startUtc.ToString("o")} - {endUtc.ToString("o")}");
+
+            var endpoint = CreateElasticSearchEndpoint();
+            var queryTemplate = CreateQueryTemplate(podName, startUtc, endUtc);
+
+            targetFile.Write($"Downloading '{podName}' to '{targetFile.Filename}'.");
+            var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
+            reconstructor.DownloadFullLog();
+
+            log.Log("Log download finished.");
+        }
+
+        private string CreateQueryTemplate(string podName, DateTime startUtc, DateTime endUtc)
+        {
+            var start = startUtc.ToString("o");
+            var end = endUtc.ToString("o");
+
+            //container_name : codex3-5 - deploymentName as stored in pod
+            // pod_namespace : codex - continuous - nolimits - tests - 1
+
+            //var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
+            var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
+            return source
+                .Replace("<STARTTIME>", start)
+                .Replace("<ENDTIME>", end)
+                .Replace("<PODNAME>", podName);
+                //.Replace("<NAMESPACENAME>", config.StorageNodesKubernetesNamespace);
+        }
+
+        private IEndpoint CreateElasticSearchEndpoint()
+        {
+            //var serviceName = "elasticsearch";
+            //var k8sNamespace = "monitoring";
+            //var address = new Address("ElasticSearchEndpoint", $"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
+
+            var address = new Address("TestnetElasticSearchEndpoint", config.ElasticSearchUrl, 443);
+            var baseUrl = "";
+
+            var username = config.GetElasticSearchUsername();
+            var password = config.GetElasticSearchPassword();
+
+            var base64Creds = Convert.ToBase64String(
+                Encoding.ASCII.GetBytes($"{username}:{password}")
+            );
+
+            var http = tools.CreateHttp(address.ToString(), client =>
+            {
+                client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
+                client.DefaultRequestHeaders.Add("Authorization", "Basic " + base64Creds);
+            });
+
+            return http.CreateEndpoint(address, baseUrl);
+        }
+
+        public class LogReconstructor
+        {
+            private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>();
+            private readonly LogFile targetFile;
+            private readonly IEndpoint endpoint;
+            private readonly string queryTemplate;
+            private const int sizeOfPage = 2000;
+            private string searchAfter = "";
+            private int lastHits = 1;
+            private ulong? lastLogLine;
+
+            public LogReconstructor(LogFile targetFile, IEndpoint endpoint, string queryTemplate)
+            {
+                this.targetFile = targetFile;
+                this.endpoint = endpoint;
+                this.queryTemplate = queryTemplate;
+            }
+
+            public void DownloadFullLog()
+            {
+                while (lastHits > 0)
+                {
+                    QueryElasticSearch();
+                    ProcessQueue();
+                }
+            }
+
+            private void QueryElasticSearch()
+            {
+                var query = queryTemplate
+                                .Replace("<SIZE>", sizeOfPage.ToString())
+                                .Replace("<SEARCHAFTER>", searchAfter);
+
+                var response = endpoint.HttpPostString<SearchResponse>("/_search", query);
+
+                lastHits = response.hits.hits.Length;
+                if (lastHits > 0)
+                {
+                    UpdateSearchAfter(response);
+                    foreach (var hit in response.hits.hits)
+                    {
+                        AddHitToQueue(hit);
+                    }
+                }
+            }
+
+            private void AddHitToQueue(SearchHitEntry hit)
+            {
+                var message = hit.fields.message.Single();
+                var number = ParseCountNumber(message);
+                if (number != null)
+                {
+                    queue.Add(new LogQueueEntry(message, number.Value));
+                }
+            }
+
+            private ulong? ParseCountNumber(string message)
+            {
+                if (string.IsNullOrEmpty(message)) return null;
+                var tokens = message.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+                if (!tokens.Any()) return null;
+                var countToken = tokens.SingleOrDefault(t => t.StartsWith("count="));
+                if (countToken == null) return null;
+                var number = countToken.Substring(6);
+                if (ulong.TryParse(number, out ulong value))
+                {
+                    return value;
+                }
+                return null;
+            }
+
+            private void UpdateSearchAfter(SearchResponse response)
+            {
+                var uniqueSearchNumbers = response.hits.hits.Select(h => h.sort.Single()).Distinct().ToList();
+                uniqueSearchNumbers.Reverse();
+
+                var searchNumber = GetSearchNumber(uniqueSearchNumbers);
+                searchAfter = $"\"search_after\": [{searchNumber}],";
+            }
+
+            private long GetSearchNumber(List<long> uniqueSearchNumbers)
+            {
+                if (uniqueSearchNumbers.Count == 1) return uniqueSearchNumbers.First();
+                return uniqueSearchNumbers.Skip(1).First();
+            }
+
+            private void ProcessQueue()
+            {
+                if (lastLogLine == null)
+                {
+                    lastLogLine = queue.Min(q => q.Number) - 1;
+                }
+
+                while (queue.Any())
+                {
+                    ulong wantedNumber = lastLogLine.Value + 1;
+
+                    DeleteOldEntries(wantedNumber);
+
+                    var currentEntry = queue.FirstOrDefault(e => e.Number == wantedNumber);
+
+                    if (currentEntry != null)
+                    {
+                        WriteEntryToFile(currentEntry);
+                        queue.Remove(currentEntry);
+                        lastLogLine = currentEntry.Number;
+                    }
+                    else
+                    {
+                        // The line number we want is not in the queue.
+                        // It will be returned by the elastic search query, some time in the future.
+                        // Stop processing the queue for now.
+                        return;
+                    }
+                }
+            }
+
+            private void WriteEntryToFile(LogQueueEntry currentEntry)
+            {
+                targetFile.Write(currentEntry.Message);
+            }
+
+            private void DeleteOldEntries(ulong wantedNumber)
+            {
+                queue.RemoveAll(e => e.Number < wantedNumber);
+            }
+
+            public class LogQueueEntry
+            {
+                public LogQueueEntry(string message, ulong number)
+                {
+                    Message = message;
+                    Number = number;
+                }
+
+                public string Message { get; }
+                public ulong Number { get; }
+            }
+
+            public class SearchResponse
+            {
+                public SearchHits hits { get; set; } = new SearchHits();
+            }
+
+            public class SearchHits
+            {
+                public SearchHitEntry[] hits { get; set; } = Array.Empty<SearchHitEntry>();
+            }
+
+            public class SearchHitEntry
+            {
+                public SearchHitFields fields { get; set; } = new SearchHitFields();
+                public long[] sort { get; set; } = Array.Empty<long>();
+            }
+
+            public class SearchHitFields
+            {
+                public string[] @timestamp { get; set; } = Array.Empty<string>();
+                public string[] message { get; set; } = Array.Empty<string>();
+            }
+        }
+    }
+}

Tools/TraceContract/Input.cs

@@ -0,0 +1,21 @@
+namespace TraceContract
+{
+    public class Input
+    {
+        public string PurchaseId
+        {
+            get
+            {
+                var v = Environment.GetEnvironmentVariable("PURCHASE_ID");
+                if (!string.IsNullOrEmpty(v)) return v;
+
+                return
+                    // expired:
+                    "a7fe97dc32216aba0cbe74b87beb3f919aa116090dd5e0d48085a1a6b0080e82";
+
+                    // started:
+                    //"066df09a3a2e2587cfd577a0e96186c915b113d02b331b06e56f808494cff2b4";
+            }
+        }
+    }
+}

Tools/TraceContract/Output.cs

@@ -0,0 +1,127 @@
+using System.Numerics;
+using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
+using Logging;
+using Utils;
+
+namespace TraceContract
+{
+    public class Output
+    {
+        private class Entry
+        {
+            public Entry(DateTime utc, string msg)
+            {
+                Utc = utc;
+                Msg = msg;
+            }
+
+            public DateTime Utc { get; }
+            public string Msg { get; }
+        }
+
+        private readonly ILog log;
+        private readonly List<Entry> entries = new();
+        private readonly string folder;
+        private readonly Input input;
+        private readonly Config config;
+
+        public Output(ILog log, Input input, Config config)
+        {
+            this.input = input;
+            this.config = config;
+
+            folder = config.GetOuputFolder();
+            Directory.CreateDirectory(folder);
+
+            var filename = Path.Combine(folder, $"contract_{input.PurchaseId}");
+            var fileLog = new FileLog(filename);
+            log.Log($"Logging to '{filename}'");
+
+            this.log = new LogSplitter(fileLog, log);
+            foreach (var pair in config.LogReplacements)
+            {
+                this.log.AddStringReplace(pair.Key, pair.Value);
+                this.log.AddStringReplace(pair.Key.ToLowerInvariant(), pair.Value);
+            }
+        }
+
+        public void LogRequestCreated(RequestEvent requestEvent)
+        {
+            Add(requestEvent.Block.Utc, $"Storage request created: '{requestEvent.Request.Request.Id}'");
+        }
+
+        public void LogRequestCancelled(RequestEvent requestEvent)
+        {
+            Add(requestEvent.Block.Utc, "Expired");
+        }
+
+        public void LogRequestFailed(RequestEvent requestEvent)
+        {
+            Add(requestEvent.Block.Utc, "Failed");
+        }
+
+        public void LogRequestFinished(RequestEvent requestEvent)
+        {
+            Add(requestEvent.Block.Utc, "Finished");
+        }
+
+        public void LogRequestStarted(RequestEvent requestEvent)
+        {
+            Add(requestEvent.Block.Utc, "Started");
+        }
+
+        public void LogSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            Add(requestEvent.Block.Utc, $"Slot filled. Index: {slotIndex} Host: '{host}'");
+        }
+
+        public void LogSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            Add(requestEvent.Block.Utc, $"Slot freed. Index: {slotIndex}");
+        }
+
+        public void LogSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            Add(requestEvent.Block.Utc, $"Slot reservations full. Index: {slotIndex}");
+        }
+
+        public void LogReserveSlotCalls(ReserveSlotFunction[] reserveSlotFunctions)
+        {
+            foreach (var call in reserveSlotFunctions) LogReserveSlotCall(call);
+        }
+
+        public void WriteContractEvents()
+        {
+            var sorted = entries.OrderBy(e => e.Utc).ToArray();
+            foreach (var e in sorted) Write(e);
+        }
+
+        public LogFile CreateNodeLogTargetFile(string node)
+        {
+            return log.CreateSubfile(node);
+        }
+
+        public void ShowOutputFiles(ILog console)
+        {
+            console.Log("Files in output folder:");
+            var files = Directory.GetFiles(folder);
+            foreach (var file in files) console.Log(file);
+        }
+
+        private void Write(Entry e)
+        {
+            log.Log($"[{Time.FormatTimestamp(e.Utc)}] {e.Msg}");
+        }
+
+        public void LogReserveSlotCall(ReserveSlotFunction call)
+        {
+            Add(call.Block.Utc, $"Reserve-slot called. Index: {call.SlotIndex} Host: '{call.FromAddress}'");
+        }
+
+        private void Add(DateTime utc, string msg)
+        {
+            entries.Add(new Entry(utc, msg));
+        }
+    }
+}

Tools/TraceContract/Program.cs

@@ -0,0 +1,98 @@
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using Core;
+using GethPlugin;
+using Logging;
+using Utils;
+
+namespace TraceContract
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            ProjectPlugin.Load<GethPlugin.GethPlugin>();
+            ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
+
+            var p = new Program();
+            p.Run();
+        }
+
+        private readonly ILog log = new ConsoleLog();
+        private readonly Input input = new();
+        private readonly Config config = new();
+        private readonly Output output;
+
+        public Program()
+        {
+            output = new(log, input, config);
+        }
+
+        private void Run()
+        {
+            try
+            {
+                TracePurchase();
+            }
+            catch (Exception exc)
+            {
+                log.Error(exc.ToString());
+            }
+        }
+
+        private void TracePurchase()
+        { 
+            Log("Setting up...");
+            var entryPoint = new EntryPoint(log, new KubernetesWorkflow.Configuration(null, TimeSpan.FromMinutes(1.0), TimeSpan.FromSeconds(10.0), "_Unused!_"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
+            entryPoint.Announce();
+            var ci = entryPoint.CreateInterface();
+            var contracts = ConnectCodexContracts(ci);
+
+            var chainTracer = new ChainTracer(log, contracts, input, output);
+            var requestTimeRange = chainTracer.TraceChainTimeline();
+
+            Log("Downloading storage nodes logs for the request timerange...");
+            DownloadStorageNodeLogs(requestTimeRange, entryPoint.Tools);
+
+            output.ShowOutputFiles(log);
+
+            entryPoint.Decommission(false, false, false);
+            Log("Done");
+        }
+
+        private ICodexContracts ConnectCodexContracts(CoreInterface ci)
+        {
+            var account = EthAccountGenerator.GenerateNew();
+            var blockCache = new BlockCache();
+            var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey);
+
+            var deployment = new CodexContractsDeployment(
+                config: new MarketplaceConfig(),
+                marketplaceAddress: config.MarketplaceAddress,
+                abi: config.Abi,
+                tokenAddress: config.TokenAddress
+            );
+            return ci.WrapCodexContractsDeployment(geth, deployment);
+        }
+
+        private void DownloadStorageNodeLogs(TimeRange requestTimeRange, IPluginTools tools)
+        {
+            var start = requestTimeRange.From - config.LogStartBeforeStorageContractStarts;
+
+            foreach (var node in config.StorageNodesKubernetesPodNames)
+            {
+                Log($"Downloading logs from '{node}'...");
+
+                var targetFile = output.CreateNodeLogTargetFile(node);
+                var downloader = new ElasticSearchLogDownloader(log, tools, config);
+                downloader.Download(targetFile, node, start, requestTimeRange.To);
+            }
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

Tools/TraceContract/TraceContract.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

cs-codex-dist-testing.sln

@@ -86,6 +86,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexClient", "ProjectPlugi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebUtils", "Framework\WebUtils\WebUtils.csproj", "{372C9E5D-5453-4D45-9948-E9324E21AD65}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TraceContract", "Tools\TraceContract\TraceContract.csproj", "{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -232,6 +234,10 @@ Global
 		{372C9E5D-5453-4D45-9948-E9324E21AD65}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{372C9E5D-5453-4D45-9948-E9324E21AD65}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{372C9E5D-5453-4D45-9948-E9324E21AD65}.Release|Any CPU.Build.0 = Release|Any CPU
+		{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -272,6 +278,7 @@ Global
 		{4648B5AA-A0A7-44BA-89BC-2FD57370943C} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{9AF12703-29AF-416D-9781-204223D6D0E5} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
 		{372C9E5D-5453-4D45-9948-E9324E21AD65} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
+		{58CDACE0-8F8D-2BB7-EA3A-0CB6A994A7F8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}