Adds assert all slots were fully reserved. Adds logs for calls to reserveslot when start fails.

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -144,7 +144,7 @@ namespace NethereumWorkflow
             return blockTimeFinder.Get(number);
         }
 
-        public BlockWithTransactions GetBlk(ulong number)
+        public BlockWithTransactions GetBlockWithTransactions(ulong number)
         {
             return Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(number)));
         }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -21,7 +21,7 @@ namespace CodexContractsPlugin
         SlotFreedEventDTO[] GetSlotFreedEvents();
         SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents();
         ProofSubmittedEventDTO[] GetProofSubmittedEvents();
-        void Do();
+        ReserveSlotFunction[] GetReserveSlotCalls();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -36,33 +36,10 @@ namespace CodexContractsPlugin
             this.gethNode = gethNode;
             this.deployment = deployment;
             BlockInterval = blockInterval;
-
-            Do();
         }
         
         public BlockInterval BlockInterval { get; }
 
-        public void Do()
-        {
-            for (ulong i = BlockInterval.From; i <= BlockInterval.To; i++)
-            {
-                var block = gethNode.GetBlk(i);
-                if (block == null) return;
-
-                foreach (var t in block.Transactions)
-                {
-                    if (t == null) continue;
-
-                    var input = t.ConvertToTransactionInput();
-                    var aaa = t.DecodeTransactionToFunctionMessage<ReserveSlotFunction>();
-                    if (aaa != null)
-                    {
-                        var a = 0;
-                    }
-                }
-            }
-        }
-
         public Request[] GetStorageRequests()
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
@@ -125,6 +102,13 @@ namespace CodexContractsPlugin
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
+        public ReserveSlotFunction[] GetReserveSlotCalls()
+        {
+            var result = new List<ReserveSlotFunction>();
+            gethNode.IterateFunctionCalls<ReserveSlotFunction>(BlockInterval, result.Add);
+            return result.ToArray();
+        }
+
         private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
         {
             var result = e.Event;

ProjectPlugins/GethPlugin/GethNode.cs

@@ -31,7 +31,8 @@ namespace GethPlugin
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
         BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
         BlockTimeEntry GetBlockForNumber(ulong number);
-        BlockWithTransactions GetBlk(ulong number);
+        void IterateFunctionCalls<TFunc>(BlockInterval blockInterval, Action<TFunc> onCall) where TFunc : FunctionMessage;
+
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -186,7 +187,22 @@ namespace GethPlugin
 
         public BlockWithTransactions GetBlk(ulong number)
         {
-            return StartInteraction().GetBlk(number);
+            return StartInteraction().GetBlockWithTransactions(number);
+        }
+
+        public void IterateFunctionCalls<TFunc>(BlockInterval blockRange, Action<TFunc> onCall) where TFunc : FunctionMessage, new()
+        {
+            var i = StartInteraction();
+            for (var blkI = blockRange.From; blkI <= blockRange.To; blkI++)
+            {
+                var blk = i.GetBlockWithTransactions(blkI);
+
+                foreach (var t in blk.Transactions)
+                {
+                    var func = t.DecodeTransactionToFunctionMessage<TFunc>();
+                    if (func != null) onCall(func);
+                }
+            }
         }
 
         protected abstract NethereumInteraction StartInteraction();

Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

@@ -39,8 +39,9 @@ namespace CodexReleaseTests.MarketTests
 
             request.WaitForStorageContractSubmitted();
             AssertContractIsOnChain(request);
+            WaitUntilSlotReservationsFull(request);
 
-            request.WaitForStorageContractStarted();
+            WaitForContractStarted(request);
             AssertContractSlotsAreFilledByHosts(request, hosts);
 
             Thread.Sleep(TimeSpan.FromSeconds(12.0));

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -254,6 +254,28 @@ namespace CodexReleaseTests.MarketTests
             }
         }
 
+        protected void WaitForContractStarted(IStoragePurchaseContract r)
+        {
+            try
+            {
+                r.WaitForStorageContractStarted();
+            }
+            catch
+            {
+                // Contract failed to start. Retrieve and log every call to ReserveSlot to identify which hosts
+                // should have filled the slot.
+
+                var requestId = r.PurchaseId.ToLowerInvariant();
+                var calls = GetContracts().GetEvents(GetTestRunTimeRange()).GetReserveSlotCalls();
+
+                Log($"Request '{requestId}' failed to start. There were {calls.Length} hosts who called reserve-slot for it:");
+                foreach (var c in calls)
+                {
+                    Log($" - Host: {c.FromAddress} RequestId: {c.RequestId.ToHex()} SlotIndex: {c.SlotIndex}");
+                }
+            }
+        }
+
         private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
         {
             var fills = GetOnChainSlotFills(hosts);
@@ -323,6 +345,39 @@ namespace CodexReleaseTests.MarketTests
             }, nameof(AssertContractIsOnChain));
         }
 
+        protected void WaitUntilSlotReservationsFull(IStoragePurchaseContract contract)
+        {
+            var requestId = contract.PurchaseId.ToLowerInvariant();
+            var slots = contract.Purchase.MinRequiredNumberOfNodes;
+
+            var timeout = TimeSpan.FromMinutes(1.0);
+            var start = DateTime.UtcNow;
+            var fullIndices = new List<ulong>();
+
+            while (DateTime.UtcNow - start < timeout)
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(3.0));
+
+                var fullEvents = GetContracts().GetEvents(GetTestRunTimeRange()).GetSlotReservationsFullEvents();
+                foreach (var e in fullEvents)
+                {
+                    if (e.RequestId.ToHex().ToLowerInvariant() == requestId)
+                    {
+                        if (!fullIndices.Contains(e.SlotIndex))
+                        {
+                            fullIndices.Add(e.SlotIndex);
+                            if (fullIndices.Count == slots) return;
+                        }
+                    }
+                }
+            }
+
+            Assert.Fail(
+                $"Slot reservations were not full after {Time.FormatDuration(timeout)}." +
+                $" Slots: {slots} Filled: {string.Join(",", fullIndices.Select(i => i.ToString()))}"
+            );
+        }
+
         protected void AssertOnChainEvents(Action<ICodexContractsEvents> onEvents, string description)
         {
             Time.Retry(() =>

Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs

@@ -57,7 +57,8 @@ namespace CodexReleaseTests.MarketTests
                 AssertContractIsOnChain(r);
             });
 
-            All(requests, r => r.WaitForStorageContractStarted());
+            All(requests, WaitUntilSlotReservationsFull);
+            All(requests, WaitForContractStarted);
 
             // for the time being, we're only interested in whether these contracts start.
             //All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts));