wip log downloading

Framework/KubernetesWorkflow/LogHandler.cs

@@ -33,7 +33,7 @@ namespace KubernetesWorkflow
             sourceLog.Log(msg);
 
             LogFile.Write(msg);
-            LogFile.WriteRaw(description);
+            LogFile.Write(description);
         }
 
         public LogFile LogFile { get; }
@@ -43,7 +43,7 @@ namespace KubernetesWorkflow
             if (line.Contains("Received JSON-RPC response")) return;
             if (line.Contains("object field not marked with serialize, skipping")) return;
 
-            LogFile.WriteRaw(line);
+            LogFile.Write(line);
         }
     }
 }

Framework/Logging/BaseLog.cs

@@ -65,7 +65,7 @@ namespace Logging
 
         public void Raw(string message)
         {
-            LogFile.WriteRaw(message);
+            LogFile.Write(message);
         }
 
         public virtual void AddStringReplace(string from, string to)

Framework/Logging/LogFile.cs

@@ -1,6 +1,4 @@
-using Utils;
-
-namespace Logging
+namespace Logging
 {
     public class LogFile
     {
@@ -16,11 +14,6 @@ namespace Logging
         public string Filename { get; private set; }
 
         public void Write(string message)
-        {
-            WriteRaw($"{GetTimestamp()} {message}");
-        }
-
-        public void WriteRaw(string message)
         {
             try
             {
@@ -50,11 +43,6 @@ namespace Logging
             }
         }
 
-        private static string GetTimestamp()
-        {
-            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
-        }
-
         private void EnsurePathExists(string filename)
         {
             var path = new FileInfo(filename).Directory!.FullName;

Framework/Logging/LogPrefixer.cs

@@ -24,17 +24,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(Prefix + message, skipFrames);
+            backingLog.Debug(GetPrefix() + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(Prefix + message);
+            backingLog.Error(GetPrefix() + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(Prefix + message);
+            backingLog.Log(GetPrefix() + message);
         }
 
         public void AddStringReplace(string from, string to)
@@ -51,5 +51,10 @@
         {
             return backingLog.GetFullName();
         }
+
+        protected virtual string GetPrefix()
+        {
+            return Prefix;
+        }
     }
 }

Framework/Logging/TimestampPrefixer.cs

@@ -0,0 +1,16 @@
+using Utils;
+
+namespace Logging
+{
+    public class TimestampPrefixer : LogPrefixer
+    {
+        public TimestampPrefixer(ILog backingLog) : base(backingLog)
+        {
+        }
+
+        protected override string GetPrefix()
+        {
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
+        }
+    }
+}

ProjectPlugins/MetricsPlugin/MetricsDownloader.cs

@@ -28,11 +28,11 @@ namespace MetricsPlugin
             var file = log.CreateSubfile("csv");
             log.Log($"Downloading metrics for {nodeName} to file {file.Filename}");
 
-            file.WriteRaw(string.Join(",", headers));
+            file.Write(string.Join(",", headers));
 
             foreach (var pair in map)
             {
-                file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
+                file.Write(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
             }
 
             return file;

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -199,7 +199,7 @@ namespace ContinuousTests
 
             private void WriteEntryToFile(LogQueueEntry currentEntry)
             {
-                targetFile.WriteRaw(currentEntry.Message);
+                targetFile.Write(currentEntry.Message);
             }
 
             private void DeleteOldEntries(ulong wantedNumber)

Tests/DistTestCore/Logs/BaseTestLog.cs

@@ -8,7 +8,7 @@ namespace DistTestCore.Logs
 
         protected BaseTestLog(ILog backingLog, string deployId)
         {
-            this.backingLog = backingLog;
+            this.backingLog = new TimestampPrefixer(backingLog);
 
             DeployId = deployId;
         }

TraceContract/ChainRequestTracker.cs

@@ -0,0 +1,103 @@
+using System.Numerics;
+using BlockchainUtils;
+using CodexContractsPlugin.ChainMonitor;
+using Utils;
+
+namespace TraceContract
+{
+    public class ChainRequestTracker : IChainStateChangeHandler
+    {
+        private readonly string requestId;
+        private readonly Output output;
+
+        public ChainRequestTracker(Output output, string requestId)
+        {
+            this.requestId = requestId.ToLowerInvariant();
+            this.output = output;
+        }
+
+        public bool IsFinished { get; private set; } = false;
+        public DateTime FinishUtc { get; private set; } = DateTime.MinValue;
+
+        public void OnError(string msg)
+        {
+        }
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent)) output.LogRequestCreated(requestEvent);
+        }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestCancelled(requestEvent);
+            }
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestFailed(requestEvent);
+            }
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                IsFinished = true;
+                FinishUtc = requestEvent.Block.Utc;
+                output.LogRequestFinished(requestEvent);
+            }
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogRequestStarted(requestEvent);
+            }
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotFilled(requestEvent, host, slotIndex);
+            }
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotFreed(requestEvent, slotIndex);
+            }
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            if (IsMyRequest(requestEvent))
+            {
+                output.LogSlotReservationsFull(requestEvent, slotIndex);
+            }
+        }
+
+        private bool IsMyRequest(RequestEvent requestEvent)
+        {
+            return requestId == requestEvent.Request.Request.Id.ToLowerInvariant();
+        }
+    }
+
+}

TraceContract/ChainTracer.cs

@@ -0,0 +1,119 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
+using Logging;
+using Nethereum.Hex.HexConvertors.Extensions;
+using Utils;
+
+namespace TraceContract
+{
+    public class ChainTracer
+    {
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly Input input;
+        private readonly Output output;
+
+        public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output)
+        {
+            this.log = log;
+            this.contracts = contracts;
+            this.input = input;
+            this.output = output;
+        }
+
+        public TimeRange TraceChainTimeline()
+        {
+            log.Log("Querying blockchain...");
+            var request = GetRequest();
+            if (request == null) throw new Exception("Failed to find the purchase in the last week of transactions.");
+
+            log.Log($"Request started at {request.Block.Utc}");
+            var contractEnd = RunToContractEnd(request);
+
+            var requestTimeline = new TimeRange(request.Block.Utc.AddMinutes(-1.0), contractEnd.AddMinutes(1.0));
+            log.Log($"Request timeline: {requestTimeline.From} -> {requestTimeline.To}");
+
+            // For this timeline, we log all the calls to reserve-slot.
+            var events = contracts.GetEvents(requestTimeline);
+            output.LogReserveSlotCalls(Filter(events.GetReserveSlotCalls()));
+
+            log.Log("Writing blockchain output...");
+            output.WriteContractEvents();
+
+            return requestTimeline;
+        }
+
+        private DateTime RunToContractEnd(Request request)
+        {
+            var utc = request.Block.Utc.AddMinutes(-1.0);
+            var tracker = new ChainRequestTracker(output, input.PurchaseId);
+            var ignoreLog = new NullLog();
+            var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false);
+
+            while (!tracker.IsFinished)
+            {
+                utc += TimeSpan.FromHours(1.0);
+                if (utc > DateTime.UtcNow)
+                {
+                    log.Log("Caught up to present moment without finding contract end.");
+                    return DateTime.UtcNow;
+                }
+
+                log.Log($"Querying up to {utc}");
+                chainState.Update(utc);
+            }
+
+            return tracker.FinishUtc;
+        }
+
+        private ReserveSlotFunction[] Filter(ReserveSlotFunction[] calls)
+        {
+            return calls.Where(c => IsThisRequest(c.RequestId)).ToArray();
+        }
+
+        private Request? GetRequest()
+        {
+            var request = FindRequest(LastHour());
+            if (request == null) request = FindRequest(LastDay());
+            if (request == null) request = FindRequest(LastWeek());
+            return request;
+        }
+
+        private Request? FindRequest(TimeRange timeRange)
+        {
+            var events = contracts.GetEvents(timeRange);
+            var requests = events.GetStorageRequests();
+
+            foreach (var r in requests)
+            {
+                if (IsThisRequest(r.RequestId))
+                {
+                    return r;
+                }
+            }
+
+            return null;
+        }
+
+        private bool IsThisRequest(byte[] requestId)
+        {
+            return requestId.ToHex().ToLowerInvariant() == input.PurchaseId.ToLowerInvariant();
+        }
+
+        private static TimeRange LastHour()
+        {
+            return new TimeRange(DateTime.UtcNow.AddHours(-1.0), DateTime.UtcNow);
+        }
+
+        private static TimeRange LastDay()
+        {
+            return new TimeRange(DateTime.UtcNow.AddDays(-1.0), DateTime.UtcNow);
+        }
+
+        private static TimeRange LastWeek()
+        {
+            return new TimeRange(DateTime.UtcNow.AddDays(-7.0), DateTime.UtcNow);
+        }
+    }
+}

TraceContract/ElasticSearchLogDownloader.cs

@@ -0,0 +1,242 @@
+using Core;
+using Logging;
+using Utils;
+using WebUtils;
+
+namespace ContinuousTests
+{
+    public class ElasticSearchLogDownloader
+    {
+        private readonly ILog log;
+        private readonly IPluginTools tools;
+        private readonly string k8SNamespace;
+
+        public ElasticSearchLogDownloader(ILog log, IPluginTools tools, string k8sNamespace)
+        {
+            this.log = log;
+            this.tools = tools;
+            k8SNamespace = k8sNamespace;
+        }
+
+        public void Download(LogFile targetFile, string containerName, DateTime startUtc, DateTime endUtc)
+        {
+            try
+            {
+                DownloadLog(targetFile, containerName, startUtc, endUtc);
+            }
+            catch (Exception ex)
+            {
+                log.Error("Failed to download log: " + ex);
+            }
+        }
+
+        private void DownloadLog(LogFile targetFile, string containerName, DateTime startUtc, DateTime endUtc)
+        {
+            log.Log($"Downloading log (from ElasticSearch) for container '{containerName}' within time range: " +
+                $"{startUtc.ToString("o")} - {endUtc.ToString("o")}");
+
+            var endpoint = CreateElasticSearchEndpoint();
+            var queryTemplate = CreateQueryTemplate(containerName, startUtc, endUtc);
+
+            targetFile.Write($"Downloading '{containerName}' to '{targetFile.Filename}'.");
+            var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
+            reconstructor.DownloadFullLog();
+
+            log.Log("Log download finished.");
+        }
+
+        private string CreateQueryTemplate(string containerName, DateTime startUtc, DateTime endUtc)
+        {
+            var start = startUtc.ToString("o");
+            var end = endUtc.ToString("o");
+
+            //container_name : codex3-5 - deploymentName as stored in pod
+            // pod_namespace : codex - continuous - nolimits - tests - 1
+
+            //var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
+            var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"container_name\": \"<CONTAINERNAME>\" } }, { \"match_phrase\": { \"pod_namespace\": \"<NAMESPACENAME>\" } } ] } } }";
+            return source
+                .Replace("<STARTTIME>", start)
+                .Replace("<ENDTIME>", end)
+                .Replace("<CONTAINERNAME>", containerName)
+                .Replace("<NAMESPACENAME>", k8SNamespace);
+        }
+
+        private IEndpoint CreateElasticSearchEndpoint()
+        {
+            var serviceName = "elasticsearch";
+            var k8sNamespace = "monitoring";
+            var address = new Address("ElasticSearchEndpoint", $"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
+            var baseUrl = "";
+
+            var http = tools.CreateHttp(address.ToString(), client =>
+            {
+                client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
+            });
+
+            return http.CreateEndpoint(address, baseUrl);
+        }
+
+        public class LogReconstructor
+        {
+            private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>();
+            private readonly LogFile targetFile;
+            private readonly IEndpoint endpoint;
+            private readonly string queryTemplate;
+            private const int sizeOfPage = 2000;
+            private string searchAfter = "";
+            private int lastHits = 1;
+            private ulong? lastLogLine;
+
+            public LogReconstructor(LogFile targetFile, IEndpoint endpoint, string queryTemplate)
+            {
+                this.targetFile = targetFile;
+                this.endpoint = endpoint;
+                this.queryTemplate = queryTemplate;
+            }
+
+            public void DownloadFullLog()
+            {
+                while (lastHits > 0)
+                {
+                    QueryElasticSearch();
+                    ProcessQueue();
+                }
+            }
+
+            private void QueryElasticSearch()
+            {
+                var query = queryTemplate
+                                .Replace("<SIZE>", sizeOfPage.ToString())
+                                .Replace("<SEARCHAFTER>", searchAfter);
+
+                var response = endpoint.HttpPostString<SearchResponse>("_search", query);
+
+                lastHits = response.hits.hits.Length;
+                if (lastHits > 0)
+                {
+                    UpdateSearchAfter(response);
+                    foreach (var hit in response.hits.hits)
+                    {
+                        AddHitToQueue(hit);
+                    }
+                }
+            }
+
+            private void AddHitToQueue(SearchHitEntry hit)
+            {
+                var message = hit.fields.message.Single();
+                var number = ParseCountNumber(message);
+                if (number != null)
+                {
+                    queue.Add(new LogQueueEntry(message, number.Value));
+                }
+            }
+
+            private ulong? ParseCountNumber(string message)
+            {
+                if (string.IsNullOrEmpty(message)) return null;
+                var tokens = message.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+                if (!tokens.Any()) return null;
+                var countToken = tokens.SingleOrDefault(t => t.StartsWith("count="));
+                if (countToken == null) return null;
+                var number = countToken.Substring(6);
+                if (ulong.TryParse(number, out ulong value))
+                {
+                    return value;
+                }
+                return null;
+            }
+
+            private void UpdateSearchAfter(SearchResponse response)
+            {
+                var uniqueSearchNumbers = response.hits.hits.Select(h => h.sort.Single()).Distinct().ToList();
+                uniqueSearchNumbers.Reverse();
+
+                var searchNumber = GetSearchNumber(uniqueSearchNumbers);
+                searchAfter = $"\"search_after\": [{searchNumber}],";
+            }
+
+            private long GetSearchNumber(List<long> uniqueSearchNumbers)
+            {
+                if (uniqueSearchNumbers.Count == 1) return uniqueSearchNumbers.First();
+                return uniqueSearchNumbers.Skip(1).First();
+            }
+
+            private void ProcessQueue()
+            {
+                if (lastLogLine == null)
+                {
+                    lastLogLine = queue.Min(q => q.Number) - 1;
+                }
+
+                while (queue.Any())
+                {
+                    ulong wantedNumber = lastLogLine.Value + 1;
+
+                    DeleteOldEntries(wantedNumber);
+
+                    var currentEntry = queue.FirstOrDefault(e => e.Number == wantedNumber);
+
+                    if (currentEntry != null)
+                    {
+                        WriteEntryToFile(currentEntry);
+                        queue.Remove(currentEntry);
+                        lastLogLine = currentEntry.Number;
+                    }
+                    else
+                    {
+                        // The line number we want is not in the queue.
+                        // It will be returned by the elastic search query, some time in the future.
+                        // Stop processing the queue for now.
+                        return;
+                    }
+                }
+            }
+
+            private void WriteEntryToFile(LogQueueEntry currentEntry)
+            {
+                targetFile.Write(currentEntry.Message);
+            }
+
+            private void DeleteOldEntries(ulong wantedNumber)
+            {
+                queue.RemoveAll(e => e.Number < wantedNumber);
+            }
+
+            public class LogQueueEntry
+            {
+                public LogQueueEntry(string message, ulong number)
+                {
+                    Message = message;
+                    Number = number;
+                }
+
+                public string Message { get; }
+                public ulong Number { get; }
+            }
+
+            public class SearchResponse
+            {
+                public SearchHits hits { get; set; } = new SearchHits();
+            }
+
+            public class SearchHits
+            {
+                public SearchHitEntry[] hits { get; set; } = Array.Empty<SearchHitEntry>();
+            }
+
+            public class SearchHitEntry
+            {
+                public SearchHitFields fields { get; set; } = new SearchHitFields();
+                public long[] sort { get; set; } = Array.Empty<long>();
+            }
+
+            public class SearchHitFields
+            {
+                public string[] @timestamp { get; set; } = Array.Empty<string>();
+                public string[] message { get; set; } = Array.Empty<string>();
+            }
+        }
+    }
+}

TraceContract/Input.cs

@@ -0,0 +1,13 @@
+namespace TraceContract
+{
+    public class Input
+    {
+        public string PurchaseId { get; } =
+            // expired:
+            "a7fe97dc32216aba0cbe74b87beb3f919aa116090dd5e0d48085a1a6b0080e82";
+
+            // started:
+            //"066df09a3a2e2587cfd577a0e96186c915b113d02b331b06e56f808494cff2b4";
+    }
+
+}

TraceContract/Output.cs

@@ -8,61 +8,112 @@ namespace TraceContract
 {
     public class Output
     {
-        private readonly ILog log;
-
-        public Output(ILog log)
+        private class Entry
         {
-            this.log = log;
+            public Entry(DateTime utc, string msg)
+            {
+                Utc = utc;
+                Msg = msg;
+            }
+
+            public DateTime Utc { get; }
+            public string Msg { get; }
+        }
+
+        private readonly ILog log;
+        private readonly List<Entry> entries = new();
+        private readonly string folder;
+        private readonly List<string> files = new();
+
+        public Output(ILog log, Input input, Config config)
+        {
+            folder = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            Directory.CreateDirectory(folder);
+
+            var filename = Path.Combine(folder, $"Contract_{input.PurchaseId}");
+            var fileLog = new FileLog(filename);
+            files.Add(fileLog.FullFilename);
+            foreach (var pair in config.LogReplacements)
+            {
+                fileLog.AddStringReplace(pair.Key, pair.Value);
+                fileLog.AddStringReplace(pair.Key.ToLowerInvariant(), pair.Value);
+            }
+
+            log.Log($"Logging to '{filename}'");
+            this.log = new LogSplitter(fileLog, log);
         }
 
         public void LogRequestCreated(RequestEvent requestEvent)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, $"Storage request created: '{requestEvent.Request.Request.Id}'");
         }
 
         public void LogRequestCancelled(RequestEvent requestEvent)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, "Expired");
         }
 
         public void LogRequestFailed(RequestEvent requestEvent)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, "Failed");
         }
 
         public void LogRequestFinished(RequestEvent requestEvent)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, "Finished");
         }
 
         public void LogRequestStarted(RequestEvent requestEvent)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, "Started");
         }
 
         public void LogSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, $"Slot filled. Index: {slotIndex} Host: '{host}'");
         }
 
         public void LogSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, $"Slot freed. Index: {slotIndex}");
         }
 
         public void LogSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            throw new NotImplementedException();
+            Add(requestEvent.Block.Utc, $"Slot reservations full. Index: {slotIndex}");
         }
 
         public void LogReserveSlotCalls(ReserveSlotFunction[] reserveSlotFunctions)
         {
-            throw new NotImplementedException();
+            foreach (var call in reserveSlotFunctions) LogReserveSlotCall(call);
         }
 
         public void WriteContractEvents()
         {
-            throw new NotImplementedException();
+            var sorted = entries.OrderBy(e => e.Utc).ToArray();
+            foreach (var e in sorted) Write(e);
+        }
+
+        private void Write(Entry e)
+        {
+            log.Log($"[{Time.FormatTimestamp(e.Utc)}] {e.Msg}");
+        }
+
+        private void LogReserveSlotCall(ReserveSlotFunction call)
+        {
+            Add(call.Block.Utc, $"Reserve-slot called. Index: {call.SlotIndex} Host: '{call.FromAddress}'");
+        }
+
+        private void Add(DateTime utc, string msg)
+        {
+            entries.Add(new Entry(utc, msg));
+        }
+
+        public LogFile CreateNodeLogTargetFile(string node)
+        {
+            var file = log.CreateSubfile(node);
+            files.Add(file.Filename);
+            return file;
         }
     }
 }

TraceContract/Program.cs

@@ -1,30 +1,21 @@
-using System.Numerics;
-using BlockchainUtils;
+using BlockchainUtils;
 using CodexContractsPlugin;
-using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
+using ContinuousTests;
 using Core;
 using GethPlugin;
 using Logging;
-using Nethereum.Hex.HexConvertors.Extensions;
 using Utils;
 
 namespace TraceContract
 {
-    public class Input
-    {
-        public string PurchaseId { get; } =
-            // expired:
-            //"a7fe97dc32216aba0cbe74b87beb3f919aa116090dd5e0d48085a1a6b0080e82";
-
-            // started:
-            "066df09a3a2e2587cfd577a0e96186c915b113d02b331b06e56f808494cff2b4";
-    }
-
     public class Program
     {
         public static void Main(string[] args)
         {
+            ProjectPlugin.Load<GethPlugin.GethPlugin>();
+            ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
+
             var p = new Program();
             p.Run();
         }
@@ -36,7 +27,7 @@ namespace TraceContract
 
         public Program()
         {
-            output = new(log);
+            output = new(log, input, config);
         }
 
         private void Run()
@@ -54,29 +45,32 @@ namespace TraceContract
         private void TracePurchase()
         { 
             Log("Setting up...");
-            var contracts = ConnectCodexContracts();
-            
-            var chainTracer = new ChainTracer(log, contracts, input, output);
-            var requestTimeRange = chainTracer.TraceChainTimeline();
-
-            Log("Done");
-        }
-
-        private ICodexContracts ConnectCodexContracts()
-        {
-            ProjectPlugin.Load<GethPlugin.GethPlugin>();
-            ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
-
             var entryPoint = new EntryPoint(log, new KubernetesWorkflow.Configuration(null, TimeSpan.FromMinutes(1.0), TimeSpan.FromSeconds(10.0), "_Unused!_"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
             entryPoint.Announce();
             var ci = entryPoint.CreateInterface();
+            var contracts = ConnectCodexContracts(ci);
+
+            var chainTracer = new ChainTracer(log, contracts, input, output);
+            var requestTimeRange = chainTracer.TraceChainTimeline();
+
+            Log("Downloading storage nodes logs for the request timerange...");
+            DownloadStorageNodeLogs(requestTimeRange, entryPoint.Tools);
+
+            // package everything
+
+            entryPoint.Decommission(false, false, false);
+            Log("Done");
+        }
+
+        private ICodexContracts ConnectCodexContracts(CoreInterface ci)
+        {
 
             var account = EthAccountGenerator.GenerateNew();
             var blockCache = new BlockCache();
             var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey);
 
             var deployment = new CodexContractsDeployment(
-                config: new CodexContractsPlugin.Marketplace.MarketplaceConfig(),
+                config: new MarketplaceConfig(),
                 marketplaceAddress: config.MarketplaceAddress,
                 abi: config.Abi,
                 tokenAddress: config.TokenAddress
@@ -84,197 +78,23 @@ namespace TraceContract
             return ci.WrapCodexContractsDeployment(geth, deployment);
         }
 
+        private void DownloadStorageNodeLogs(TimeRange requestTimeRange, IPluginTools tools)
+        {
+            var start = requestTimeRange.From - config.LogStartBeforeStorageContractStarts;
+
+            foreach (var node in config.StorageNodesKubernetesContainerNames)
+            {
+                Log($"Downloading logs from '{node}'...");
+
+                var targetFile = output.CreateNodeLogTargetFile(node);
+                var downloader = new ElasticSearchLogDownloader(log, tools, config.StorageNodesKubernetesNamespace);
+                downloader.Download(targetFile, node, start, requestTimeRange.To);
+            }
+        }
+
         private void Log(string msg)
         {
             log.Log(msg);
         }
     }
-
-    public class ChainTracer
-    {
-        private readonly ILog log;
-        private readonly ICodexContracts contracts;
-        private readonly Input input;
-        private readonly Output output;
-
-        public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output)
-        {
-            this.log = log;
-            this.contracts = contracts;
-            this.input = input;
-            this.output = output;
-        }
-
-        public TimeRange TraceChainTimeline()
-        {
-            var request = GetRequest();
-            if (request == null) throw new Exception("Failed to find the purchase in the last week of transactions.");
-
-            var utc = request.Block.Utc.AddMinutes(-1.0);
-            var tracker = new ChainRequestTracker(output, input.PurchaseId);
-            var ignoreLog = new NullLog();
-            var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false);
-
-            while (!tracker.IsFinished)
-            {
-                utc += TimeSpan.FromHours(1.0);
-                chainState.Update(utc);
-            }
-
-            var requestTimeline = new TimeRange(request.Block.Utc.AddMinutes(-1.0), tracker.FinishUtc.AddMinutes(1.0));
-
-            // For this timeline, we log all the calls to reserve-slot.
-            var events = contracts.GetEvents(requestTimeline);
-            output.LogReserveSlotCalls(Filter(events.GetReserveSlotCalls()));
-
-            output.WriteContractEvents();
-
-            return requestTimeline;
-        }
-
-        private ReserveSlotFunction[] Filter(ReserveSlotFunction[] calls)
-        {
-            return calls.Where(c => IsThisRequest(c.RequestId)).ToArray();
-        }
-
-        private Request? GetRequest()
-        {
-            var request = FindRequest(LastHour());
-            if (request == null) request = FindRequest(LastDay());
-            if (request == null) request = FindRequest(LastWeek());
-            return request;
-        }
-
-        private Request? FindRequest(TimeRange timeRange)
-        {
-            var events = contracts.GetEvents(timeRange);
-            var requests = events.GetStorageRequests();
-
-            foreach (var r in requests)
-            {
-                if (IsThisRequest(r.RequestId))
-                {
-                    return r;
-                }
-            }
-
-            return null;
-        }
-
-        private bool IsThisRequest(byte[] requestId)
-        {
-            return requestId.ToHex().ToLowerInvariant() == input.PurchaseId.ToLowerInvariant();
-        }
-
-        private static TimeRange LastHour()
-        {
-            return new TimeRange(DateTime.UtcNow.AddHours(-1.0), DateTime.UtcNow);
-        }
-
-        private static TimeRange LastDay()
-        {
-            return new TimeRange(DateTime.UtcNow.AddDays(-1.0), DateTime.UtcNow);
-        }
-
-        private static TimeRange LastWeek()
-        {
-            return new TimeRange(DateTime.UtcNow.AddDays(-7.0), DateTime.UtcNow);
-        }
-    }
-
-    public class ChainRequestTracker : IChainStateChangeHandler
-    {
-        private readonly string requestId;
-        private readonly Output output;
-
-        public ChainRequestTracker(Output output, string requestId)
-        {
-            this.requestId = requestId.ToLowerInvariant();
-            this.output = output;
-        }
-
-        public bool IsFinished { get; private set; } = false;
-        public DateTime FinishUtc { get; private set; } = DateTime.MinValue;
-
-        public void OnError(string msg)
-        {
-        }
-
-        public void OnNewRequest(RequestEvent requestEvent)
-        {
-            if (IsMyRequest(requestEvent)) output.LogRequestCreated(requestEvent);
-        }
-
-        public void OnProofSubmitted(BlockTimeEntry block, string id)
-        {
-        }
-
-        public void OnRequestCancelled(RequestEvent requestEvent)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                IsFinished = true;
-                FinishUtc = requestEvent.Block.Utc;
-                output.LogRequestCancelled(requestEvent);
-            }
-        }
-
-        public void OnRequestFailed(RequestEvent requestEvent)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                IsFinished = true;
-                FinishUtc = requestEvent.Block.Utc;
-                output.LogRequestFailed(requestEvent);
-            }
-        }
-
-        public void OnRequestFinished(RequestEvent requestEvent)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                IsFinished = true;
-                FinishUtc = requestEvent.Block.Utc;
-                output.LogRequestFinished(requestEvent);
-            }
-        }
-
-        public void OnRequestFulfilled(RequestEvent requestEvent)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                output.LogRequestStarted(requestEvent);
-            }
-        }
-
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                output.LogSlotFilled(requestEvent, host, slotIndex);
-            }
-        }
-
-        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                output.LogSlotFreed(requestEvent, slotIndex);
-            }
-        }
-
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            if (IsMyRequest(requestEvent))
-            {
-                output.LogSlotReservationsFull(requestEvent, slotIndex);
-            }
-        }
-
-        private bool IsMyRequest(RequestEvent requestEvent)
-        {
-            return requestId == requestEvent.Request.Request.Id.ToLowerInvariant();
-        }
-    }
-
 }