Merge branch 'master' into feature/multi-codex-folder-saver

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -27,7 +27,7 @@ namespace KubernetesWorkflow.Types
         public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
-            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
+            if (addresses.Length == 0) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
             return select.Address;

ProjectPlugins/CodexClient/CodexNode.cs

@@ -299,7 +299,7 @@ namespace CodexClient
 
         private void InitializePeerNodeId()
         {
-            var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online");
+            var debugInfo = codexAccess.GetDebugInfo();
             if (!debugInfo.Version.IsValid())
             {
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,6 +1,7 @@
 using BlockchainUtils;
 using CodexContractsPlugin.Marketplace;
 using Logging;
+using Nethereum.Hex.HexConvertors.Extensions;
 using System.Numerics;
 using Utils;
 
@@ -13,7 +14,7 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFulfilled(RequestEvent requestEvent);
         void OnRequestCancelled(RequestEvent requestEvent);
         void OnRequestFailed(RequestEvent requestEvent);
-        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
+        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
         void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
         void OnProofSubmitted(BlockTimeEntry block, string id);
@@ -118,7 +119,17 @@ namespace CodexContractsPlugin.ChainMonitor
         private void ApplyEvent(Request request)
         {
             if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
-                throw new Exception("Received NewRequest event for id that already exists.");
+            {
+                var r = FindRequest(request);
+                if (r == null) throw new Exception("ChainState is inconsistent. Received already-known requestId that's not known.");
+
+                if (request.Block.BlockNumber != r.Request.Block.BlockNumber) throw new Exception("Same request found in different blocks.");
+                if (request.Client != r.Request.Client) throw new Exception("Same request belongs to different clients.");
+                if (request.Content.Cid.ToHex() != r.Request.Content.Cid.ToHex()) throw new Exception("Same request has different CIDs.");
+
+                log.Log("Received the same request-creation event multiple times.");
+                return;
+            }
 
             var newRequest = new ChainStateRequest(log, request, RequestState.New);
             requests.Add(newRequest);
@@ -154,16 +165,18 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             var r = FindRequest(@event);
             if (r == null) return;
-            r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
+            var slotIndex = (int)@event.SlotIndex;
+            var isRepair = !r.Hosts.IsFilled(slotIndex) && r.Hosts.WasPreviouslyFilled(slotIndex);
+            r.Hosts.HostFillsSlot(@event.Host, slotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
-            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex);
+            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex, isRepair);
         }
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
             var r = FindRequest(@event);
             if (r == null) return;
-            r.Hosts.RemoveHost((int)@event.SlotIndex);
+            r.Hosts.SlotFreed((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -38,9 +38,9 @@ namespace CodexContractsPlugin.ChainMonitor
             foreach (var handler in Handlers) handler.OnRequestFulfilled(requestEvent);
         }
 
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
-            foreach (var handler in Handlers) handler.OnSlotFilled(requestEvent, host, slotIndex);
+            foreach (var handler in Handlers) handler.OnSlotFilled(requestEvent, host, slotIndex, isRepair);
         }
 
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -55,13 +55,25 @@ namespace CodexContractsPlugin.ChainMonitor
     public class RequestHosts
     {
         private readonly Dictionary<int, EthAddress> hosts = new Dictionary<int, EthAddress>();
+        private readonly List<int> filled = new List<int>();
 
-        public void Add(EthAddress host, int index)
+        public void HostFillsSlot(EthAddress host, int index)
         {
             hosts.Add(index, host);
+            filled.Add(index);
+        }
+
+        public bool IsFilled(int index)
+        {
+            return hosts.ContainsKey(index);
+        }
+
+        public bool WasPreviouslyFilled(int index)
+        {
+            return filled.Contains(index);
         }
         
-        public void RemoveHost(int index)
+        public void SlotFreed(int index)
         {
             hosts.Remove(index);
         }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -26,7 +26,7 @@ namespace CodexContractsPlugin.ChainMonitor
         {
         }
 
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
         }
 

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -35,12 +35,15 @@ namespace CodexContractsPlugin
             var container = containers.Containers[0];
 
             Log("Container started.");
+            var watcher = workflow.CreateCrashWatcher(container);
+            watcher.Start();
 
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
 
                 workflow.Stop(containers, waitTillStopped: false);
+                watcher.Stop();
                 Log("Container stopped.");
                 return result;
             }

ProjectPlugins/CodexPlugin/CodexDockerImage.cs

@@ -2,7 +2,7 @@
 {
     public class CodexDockerImage
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-3e17207-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-e324ac8-dist-tests";
 
         public static string Override { get; set; } = string.Empty;
 

Tests/CodexReleaseTests/DataTests/DecodeTest.cs

@@ -1,6 +1,4 @@
-using System.Security.Cryptography;
-using CodexReleaseTests.Utils;
-using Nethereum.JsonRpc.Client;
+using CodexReleaseTests.Utils;
 using NUnit.Framework;
 using Utils;
 

Tests/DistTestCore/DistTest.cs

@@ -179,16 +179,34 @@ namespace DistTestCore
 
         private IWebCallTimeSet GetWebCallTimeSet()
         {
+            if (IsRunningInCluster())
+            {
+                Log(" > Detected we're running in the cluster. Using long webCall timeset.");
+                return new LongWebCallTimeSet();
+            }
+
             if (ShouldUseLongTimeouts()) return new LongWebCallTimeSet();
             return new DefaultWebCallTimeSet();
         }
 
         private IK8sTimeSet GetK8sTimeSet()
         {
+            if (IsRunningInCluster())
+            {
+                Log(" > Detected we're running in the cluster. Using long kubernetes timeset.");
+                return new LongK8sTimeSet();
+            }
+
             if (ShouldUseLongTimeouts()) return new LongK8sTimeSet();
             return new DefaultK8sTimeSet();
         }
 
+        private bool IsRunningInCluster()
+        {
+            var testType = Environment.GetEnvironmentVariable("TEST_TYPE");
+            return testType == "release-tests";
+        }
+
         private bool ShouldWaitForCleanup()
         {
             return CurrentTestMethodHasAttribute<WaitForCleanupAttribute>();
@@ -256,7 +274,7 @@ namespace DistTestCore
 
         private string GetCurrentTestName()
         {
-            return $"[{TestContext.CurrentContext.Test.Name}]";
+            return $"[{NameUtils.GetTestMethodName()}]";
         }
 
         public DistTestResult GetTestResult()

Tools/BiblioTech/AdminChecker.cs

@@ -35,6 +35,7 @@ namespace BiblioTech
 
         public async Task SendInAdminChannel(string[] lines)
         {
+            if (adminChannel == null) return;
             var chunker = new LineChunker(lines);
             var chunks = chunker.GetChunks();
             if (!chunks.Any()) return;

Tools/BiblioTech/BaseCommand.cs

@@ -20,7 +20,9 @@ namespace BiblioTech
                 Program.Log.Log($"Responding to '{Name}'");
                 var context = new CommandContext(command, command.Data.Options);
                 await command.RespondAsync(StartingMessage, ephemeral: IsEphemeral(context));
-                await Invoke(context);
+
+                // Fire and forget invocation handler. Return SlashCommandHandler immediately.
+                _ = Invoke(context);
             }
             catch (Exception ex)
             {

Tools/BiblioTech/BaseGethCommand.cs

@@ -8,9 +8,13 @@ namespace BiblioTech
     {
         protected override async Task Invoke(CommandContext context)
         {
-            var gethConnector = GethConnector.GethConnector.Initialize(Program.Log);
+            var gethConnector = GetGeth();
+            if (gethConnector == null)
+            {
+                await context.Followup("Blockchain operations are (temporarily) unavailable.");
+                return;
+            }
 
-            if (gethConnector == null) return;
             var gethNode = gethConnector.GethNode;
             var contracts = gethConnector.CodexContracts;
 
@@ -23,6 +27,19 @@ namespace BiblioTech
             await Execute(context, gethNode, contracts);
         }
 
+        private GethConnector.GethConnector? GetGeth()
+        {
+            try
+            {
+                return GethConnector.GethConnector.Initialize(Program.Log);
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error("Failed to initialize geth connector: " + ex);
+                return null;
+            }
+        }
+
         protected abstract Task Execute(CommandContext context, IGethNode gethNode, ICodexContracts contracts);
     }
 }

Tools/BiblioTech/CodexChecking/CheckRepo.cs

@@ -1,16 +1,20 @@
-using Newtonsoft.Json;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
 
 namespace BiblioTech.CodexChecking
 {
     public class CheckRepo
     {
         private const string modelFilename = "model.json";
+        private readonly ILog log;
         private readonly Configuration config;
         private readonly object _lock = new object();
         private CheckRepoModel? model = null;
 
-        public CheckRepo(Configuration config)
+        public CheckRepo(ILog log, Configuration config)
         {
+            this.log = log;
             this.config = config;
         }
 
@@ -18,20 +22,32 @@ namespace BiblioTech.CodexChecking
         {
             lock (_lock)
             {
-                if (model == null) LoadModel();
-
-                var existing = model.Reports.SingleOrDefault(r  => r.UserId == userId);
-                if (existing == null)
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                try
                 {
-                    var newEntry = new CheckReport
+                    if (model == null) LoadModel();
+
+                    var existing = model.Reports.SingleOrDefault(r => r.UserId == userId);
+                    if (existing == null)
                     {
-                        UserId = userId,
-                    };
-                    model.Reports.Add(newEntry);
-                    SaveChanges();
-                    return newEntry;
+                        var newEntry = new CheckReport
+                        {
+                            UserId = userId,
+                        };
+                        model.Reports.Add(newEntry);
+                        SaveChanges();
+                        return newEntry;
+                    }
+                    return existing;
+                }
+                finally
+                {
+                    var elapsed = sw.Elapsed;
+                    if (elapsed > TimeSpan.FromMilliseconds(500))
+                    {
+                        log.Log($"Warning {nameof(GetOrCreate)} took {Time.FormatDuration(elapsed)}");
+                    }
                 }
-                return existing;
             }
         }
 

Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs

@@ -15,6 +15,7 @@ namespace BiblioTech.CodexChecking
         Task CouldNotDownloadCid();
         Task GiveCidToUser(string cid);
         Task GiveDataFileToUser(string fileContent);
+        Task CodexUnavailable();
 
         Task ToAdminChannel(string msg);
     }
@@ -44,6 +45,12 @@ namespace BiblioTech.CodexChecking
             }
 
             var cid = UploadData(check.UniqueData);
+            if (cid == null)
+            {
+                await handler.CodexUnavailable();
+                return;
+            }
+
             await handler.GiveCidToUser(cid);
         }
 
@@ -96,7 +103,12 @@ namespace BiblioTech.CodexChecking
 
             if (IsManifestLengthCompatible(handler, check, manifest))
             {
-                if (IsContentCorrect(handler, check, receivedCid))
+                var correct = IsContentCorrect(handler, check, receivedCid);
+                if (!correct.HasValue) {
+                    await handler.CodexUnavailable();
+                    return;
+                }
+                if (correct.Value)
                 {
                     await CheckNowCompleted(handler, check, userId, "UploadCheck");
                     return;
@@ -120,7 +132,7 @@ namespace BiblioTech.CodexChecking
                 check.CompletedUtc < expiry;
         }
 
-        private string UploadData(string uniqueData)
+        private string? UploadData(string uniqueData)
         {
             var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString());
 
@@ -172,7 +184,7 @@ namespace BiblioTech.CodexChecking
                 manifestLength < (dataLength + 1);
         }
 
-        private bool IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid)
+        private bool? IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid)
         {
             try
             {
@@ -190,6 +202,8 @@ namespace BiblioTech.CodexChecking
                     }
                 });
 
+                if (content == null) return null;
+
                 Log($"Checking content: content={content},check={check.UniqueData}");
                 return content == check.UniqueData;
             }

Tools/BiblioTech/CodexChecking/CodexWrapper.cs

@@ -21,32 +21,72 @@ namespace BiblioTech.CodexChecking
 
             var httpFactory = CreateHttpFactory();
             factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath);
+
+            Task.Run(CheckCodexNode);
         }
 
-        public void OnCodex(Action<ICodexNode> action)
+        public T? OnCodex<T>(Func<ICodexNode, T> func) where T : class
         {
             lock (codexLock)
             {
-                action(Get());
+                if (currentCodexNode == null) return null;
+                return func(currentCodexNode);
             }
         }
 
-        public T OnCodex<T>(Func<ICodexNode, T> func)
+        private void CheckCodexNode()
         {
-            lock (codexLock)
+            Thread.Sleep(TimeSpan.FromSeconds(10.0));
+
+            while (true)
             {
-                return func(Get());
+                lock (codexLock)
+                {
+                    var newNode = GetNewCodexNode();
+                    if (newNode != null && currentCodexNode == null) ShowConnectionRestored();
+                    if (newNode == null && currentCodexNode != null) ShowConnectionLost();
+                    currentCodexNode = newNode;
+                }
+
+                Thread.Sleep(TimeSpan.FromMinutes(15.0));
             }
         }
 
-        private ICodexNode Get()
+        private ICodexNode? GetNewCodexNode()
         {
-            if (currentCodexNode == null)
+            try
             {
-                currentCodexNode = CreateCodex();
-            }
+                if (currentCodexNode != null)
+                {
+                    try
+                    {
+                        // Current instance is responsive? Keep it.
+                        var info = currentCodexNode.GetDebugInfo();
+                        if (info != null && info.Version != null &&
+                            !string.IsNullOrEmpty(info.Version.Revision)) return currentCodexNode;
+                    }
+                    catch
+                    {
+                    }
+                }
 
-            return currentCodexNode;
+                return CreateCodex();
+            }
+            catch (Exception ex)
+            {
+                log.Error("Exception when trying to check codex node: " + ex.Message);
+                return null;
+            }
+        }
+
+        private void ShowConnectionLost()
+        {
+            Program.AdminChecker.SendInAdminChannel("Codex node connection lost.");
+        }
+
+        private void ShowConnectionRestored()
+        {
+            Program.AdminChecker.SendInAdminChannel("Codex node connection restored.");
         }
 
         private ICodexNode CreateCodex()
@@ -70,16 +110,34 @@ namespace BiblioTech.CodexChecking
         {
             if (string.IsNullOrEmpty(config.CodexEndpointAuth) || !config.CodexEndpointAuth.Contains(":"))
             {
-                return new HttpFactory(log);
+                return new HttpFactory(log, new SnappyTimeSet());
             }
 
             var tokens = config.CodexEndpointAuth.Split(':');
             if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
 
-            return new HttpFactory(log, onClientCreated: client =>
+            return new HttpFactory(log, new SnappyTimeSet(), onClientCreated: client =>
             {
                 client.SetBasicAuthentication(tokens[0], tokens[1]);
             });
         }
+
+        public class SnappyTimeSet : IWebCallTimeSet
+        {
+            public TimeSpan HttpCallRetryDelay()
+            {
+                return TimeSpan.FromSeconds(1.0);
+            }
+
+            public TimeSpan HttpCallTimeout()
+            {
+                return TimeSpan.FromSeconds(3.0);
+            }
+
+            public TimeSpan HttpRetryTimeout()
+            {
+                return TimeSpan.FromSeconds(12.0);
+            }
+        }
     }
 }

Tools/BiblioTech/Commands/CheckDownloadCommand.cs

@@ -18,7 +18,7 @@ namespace BiblioTech.Commands
         }
 
         public override string Name => "checkdownload";
-        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string StartingMessage => "Connecting to the testnet... Please be patient... " + RandomBusyMessage.Get();
         public override string Description => "Checks the download connectivity of your Codex node.";
         public override CommandOption[] Options => [contentOption];
 

Tools/BiblioTech/Commands/CheckResponseHandler.cs

@@ -26,6 +26,11 @@ namespace BiblioTech.Commands
             await context.Followup("Could not download the CID.");
         }
 
+        public async Task CodexUnavailable()
+        {
+            await context.Followup("Couldn't perform check: Our Codex node appears unavailable. Try again later?");
+        }
+
         public async Task GiveCidToUser(string cid)
         {
             await context.Followup(

Tools/BiblioTech/Commands/CheckUploadCommand.cs

@@ -18,7 +18,7 @@ namespace BiblioTech.Commands
         }
 
         public override string Name => "checkupload";
-        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string StartingMessage => "Connecting to the testnet... Please be patient... " + RandomBusyMessage.Get();
         public override string Description => "Checks the upload connectivity of your Codex node.";
         public override CommandOption[] Options => [cidOption];
 

Tools/BiblioTech/Program.cs

@@ -88,7 +88,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checkRepo = new CheckRepo(Config);
+            var checkRepo = new CheckRepo(Log, Config);
             var codexWrapper = new CodexWrapper(Log, Config);
             var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper);
             var notifyCommand = new NotifyCommand();

Tools/MarketInsights/ContributionBuilder.cs

@@ -1,6 +1,5 @@
 using BlockchainUtils;
 using CodexContractsPlugin.ChainMonitor;
-using GethPlugin;
 using Logging;
 using System.Numerics;
 using Utils;
@@ -47,7 +46,7 @@ namespace MarketInsights
             AddRequestToAverage(segment.Started, requestEvent);
         }
 
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
         }
 

Tools/TestNetRewarder/EmojiMaps.cs

@@ -78,6 +78,7 @@
         public string NewRequest => "🌱";
         public string Started => "🌳";
         public string SlotFilled => "🟢";
+        public string SlotRepaired => "♻";
         public string SlotFreed => "⭕";
         public string SlotReservationsFull => "☑️";
         public string Finished => "✅";

Tools/TestNetRewarder/EventsFormatter.cs

@@ -1,8 +1,7 @@
 using BlockchainUtils;
-using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
 using DiscordRewards;
-using GethPlugin;
 using System.Globalization;
 using System.Numerics;
 using Utils;
@@ -16,10 +15,12 @@ namespace TestNetRewarder
         private readonly List<string> errors = new List<string>();
         private readonly EmojiMaps emojiMaps = new EmojiMaps();
         private readonly Configuration config;
+        private readonly string periodDuration;
 
-        public EventsFormatter(Configuration config)
+        public EventsFormatter(Configuration config, MarketplaceConfig marketplaceConfig)
         {
             this.config = config;
+            periodDuration = Time.FormatDuration(marketplaceConfig.PeriodDuration);
         }
 
         public ChainEventMessage[] GetInitializationEvents(Configuration config)
@@ -49,7 +50,7 @@ namespace TestNetRewarder
         public void OnNewRequest(RequestEvent requestEvent)
         {
             var request = requestEvent.Request;
-            AddRequestBlock(requestEvent, $"{emojiMaps.NewRequest} New Request",
+            AddRequestBlock(requestEvent, emojiMaps.NewRequest, "New Request",
                 $"Client: {request.Client}",
                 $"Content: {BytesToHexString(request.Request.Content.Cid)}",
                 $"Duration: {BigIntToDuration(request.Request.Ask.Duration)}",
@@ -58,33 +59,34 @@ namespace TestNetRewarder
                 $"PricePerBytePerSecond: {BitIntToTestTokens(request.Request.Ask.PricePerBytePerSecond)}",
                 $"Number of Slots: {request.Request.Ask.Slots}",
                 $"Slot Tolerance: {request.Request.Ask.MaxSlotLoss}",
-                $"Slot Size: {BigIntToByteSize(request.Request.Ask.SlotSize)}"
+                $"Slot Size: {BigIntToByteSize(request.Request.Ask.SlotSize)}",
+                $"Proof Probability: 1 / {request.Request.Ask.ProofProbability} every {periodDuration}"
             );
         }
 
         public void OnRequestCancelled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Cancelled} Cancelled");
+            AddRequestBlock(requestEvent, emojiMaps.Cancelled, "Cancelled");
         }
 
         public void OnRequestFailed(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Failed} Failed");
+            AddRequestBlock(requestEvent, emojiMaps.Failed, "Failed");
         }
 
         public void OnRequestFinished(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Finished} Finished");
+            AddRequestBlock(requestEvent, emojiMaps.Finished, "Finished");
         }
 
         public void OnRequestFulfilled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Started} Started");
+            AddRequestBlock(requestEvent, emojiMaps.Started, "Started");
         }
 
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFilled} Slot Filled",
+            AddRequestBlock(requestEvent, GetSlotFilledIcon(isRepair), GetSlotFilledTitle(isRepair),
                 $"Host: {host}",
                 $"Slot Index: {slotIndex}"
             );
@@ -92,14 +94,14 @@ namespace TestNetRewarder
 
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFreed} Slot Freed",
+            AddRequestBlock(requestEvent, emojiMaps.SlotFreed, "Slot Freed",
                 $"Slot Index: {slotIndex}"
             );
         }
 
         public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotReservationsFull} Slot Reservations Full",
+            AddRequestBlock(requestEvent, emojiMaps.SlotReservationsFull, "Slot Reservations Full",
                 $"Slot Index: {slotIndex}"
             );
         }
@@ -133,6 +135,18 @@ namespace TestNetRewarder
             AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray());
         }
 
+        private string GetSlotFilledIcon(bool isRepair)
+        {
+            if (isRepair) return emojiMaps.SlotRepaired;
+            return emojiMaps.SlotFilled;
+        }
+
+        private string GetSlotFilledTitle(bool isRepair)
+        {
+            if (isRepair) return $"Slot Repaired";
+            return $"Slot Filled";
+        }
+
         private void AddMissedProofDetails(List<string> lines, PeriodReport[] reports)
         {
             var reportsWithMissedProofs = reports.Where(r => r.MissedProofs.Length > 0).ToArray();
@@ -168,10 +182,10 @@ namespace TestNetRewarder
             lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})");
         }
 
-        private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
+        private void AddRequestBlock(RequestEvent requestEvent, string icon, string eventName, params string[] content)
         {
             var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
-            var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}";
+            var title = $"{blockNumber} {icon} **{eventName}** {FormatRequestId(requestEvent)}";
             AddBlock(requestEvent.Block.BlockNumber, title, content);
         }
 

Tools/TestNetRewarder/Processor.cs

@@ -25,7 +25,7 @@ namespace TestNetRewarder
             if (config.ProofReportHours < 1) throw new Exception("ProofReportHours must be one or greater");
 
             builder = new RequestBuilder();
-            eventsFormatter = new EventsFormatter(config);
+            eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config);
 
             chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc,
                 doProofPeriodMonitoring: config.ShowProofPeriodReports > 0);

Tools/TraceContract/ChainRequestTracker.cs

@@ -70,11 +70,11 @@ namespace TraceContract
             }
         }
 
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
             if (IsMyRequest(requestEvent))
             {
-                output.LogSlotFilled(requestEvent, host, slotIndex);
+                output.LogSlotFilled(requestEvent, host, slotIndex, isRepair);
             }
         }
 

Tools/TraceContract/ElasticSearchLogDownloader.cs

@@ -40,7 +40,7 @@ namespace TraceContract
             var queryTemplate = CreateQueryTemplate(podName, startUtc, endUtc);
 
             targetFile.Write($"Downloading '{podName}' to '{targetFile.Filename}'.");
-            var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
+            var reconstructor = new LogReconstructor(log, targetFile, endpoint, queryTemplate);
             reconstructor.DownloadFullLog();
 
             log.Log("Log download finished.");
@@ -91,6 +91,7 @@ namespace TraceContract
         public class LogReconstructor
         {
             private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>();
+            private readonly ILog log;
             private readonly LogFile targetFile;
             private readonly IEndpoint endpoint;
             private readonly string queryTemplate;
@@ -98,9 +99,11 @@ namespace TraceContract
             private string searchAfter = "";
             private int lastHits = 1;
             private ulong? lastLogLine;
+            private uint linesWritten = 0;
 
-            public LogReconstructor(LogFile targetFile, IEndpoint endpoint, string queryTemplate)
+            public LogReconstructor(ILog log, LogFile targetFile, IEndpoint endpoint, string queryTemplate)
             {
+                this.log = log;
                 this.targetFile = targetFile;
                 this.endpoint = endpoint;
                 this.queryTemplate = queryTemplate;
@@ -113,6 +116,8 @@ namespace TraceContract
                     QueryElasticSearch();
                     ProcessQueue();
                 }
+
+                log.Log($"{linesWritten} lines written.");
             }
 
             private void QueryElasticSearch()
@@ -124,6 +129,8 @@ namespace TraceContract
                 var response = endpoint.HttpPostString<SearchResponse>("/_search", query);
 
                 lastHits = response.hits.hits.Length;
+                log.Log($"pageSize: {sizeOfPage} after: {searchAfter} -> {lastHits} hits");
+
                 if (lastHits > 0)
                 {
                     UpdateSearchAfter(response);
@@ -176,7 +183,7 @@ namespace TraceContract
 
             private void ProcessQueue()
             {
-                if (lastLogLine == null)
+                if (lastLogLine == null && queue.Any())
                 {
                     lastLogLine = queue.Min(q => q.Number) - 1;
                 }
@@ -208,6 +215,7 @@ namespace TraceContract
             private void WriteEntryToFile(LogQueueEntry currentEntry)
             {
                 targetFile.Write(currentEntry.Message);
+                linesWritten++;
             }
 
             private void DeleteOldEntries(ulong wantedNumber)

Tools/TraceContract/Output.cs

@@ -1,4 +1,5 @@
 using System.Numerics;
+using BlockchainUtils;
 using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using Logging;
@@ -10,13 +11,13 @@ namespace TraceContract
     {
         private class Entry
         {
-            public Entry(DateTime utc, string msg)
+            public Entry(BlockTimeEntry blk, string msg)
             {
-                Utc = utc;
+                Blk = blk;
                 Msg = msg;
             }
 
-            public DateTime Utc { get; }
+            public BlockTimeEntry Blk { get; }
             public string Msg { get; }
         }
 
@@ -48,47 +49,47 @@ namespace TraceContract
 
         public void LogRequestCreated(RequestEvent requestEvent)
         {
-            Add(requestEvent.Block.Utc, $"Storage request created: '{requestEvent.Request.Request.Id}'");
+            Add(requestEvent.Block, $"Storage request created: '{requestEvent.Request.Request.Id}'");
         }
 
         public void LogRequestCancelled(RequestEvent requestEvent)
         {
-            Add(requestEvent.Block.Utc, "Expired");
+            Add(requestEvent.Block, "Expired");
         }
 
         public void LogRequestFailed(RequestEvent requestEvent)
         {
-            Add(requestEvent.Block.Utc, "Failed");
+            Add(requestEvent.Block, "Failed");
         }
 
         public void LogRequestFinished(RequestEvent requestEvent)
         {
-            Add(requestEvent.Block.Utc, "Finished");
+            Add(requestEvent.Block, "Finished");
         }
 
         public void LogRequestStarted(RequestEvent requestEvent)
         {
-            Add(requestEvent.Block.Utc, "Started");
+            Add(requestEvent.Block, "Started");
         }
 
-        public void LogSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        public void LogSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
         {
-            Add(requestEvent.Block.Utc, $"Slot filled. Index: {slotIndex} Host: '{host}'");
+            Add(requestEvent.Block, $"Slot filled. Index: {slotIndex} Host: '{host}' isRepair: {isRepair}");
         }
 
         public void LogSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            Add(requestEvent.Block.Utc, $"Slot freed. Index: {slotIndex}");
+            Add(requestEvent.Block, $"Slot freed. Index: {slotIndex}");
         }
 
         public void LogSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            Add(requestEvent.Block.Utc, $"Slot reservations full. Index: {slotIndex}");
+            Add(requestEvent.Block, $"Slot reservations full. Index: {slotIndex}");
         }
 
         public void WriteContractEvents()
         {
-            var sorted = entries.OrderBy(e => e.Utc).ToArray();
+            var sorted = entries.OrderBy(e => e.Blk.Utc).ToArray();
             foreach (var e in sorted) Write(e);
         }
 
@@ -106,17 +107,17 @@ namespace TraceContract
 
         private void Write(Entry e)
         {
-            log.Log($"[{Time.FormatTimestamp(e.Utc)}] {e.Msg}");
+            log.Log($"Block: {e.Blk.BlockNumber} [{Time.FormatTimestamp(e.Blk.Utc)}] {e.Msg}");
         }
 
         public void LogReserveSlotCall(ReserveSlotFunction call)
         {
-            Add(call.Block.Utc, $"Reserve-slot called. Index: {call.SlotIndex} Host: '{call.FromAddress}'");
+            Add(call.Block, $"Reserve-slot called. Block: {call.Block.BlockNumber} Index: {call.SlotIndex} Host: '{call.FromAddress}'");
         }
 
-        private void Add(DateTime utc, string msg)
+        private void Add(BlockTimeEntry blk, string msg)
         {
-            entries.Add(new Entry(utc, msg));
+            entries.Add(new Entry(blk, msg));
         }
     }
 }