very very WIP for how to download complete log from elastic search

Framework/Core/CoreInterface.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using Utils;
 
 namespace Core
 {
@@ -57,6 +58,17 @@ namespace Core
             var workflow = entryPoint.Tools.CreateWorkflow();
             return workflow.ExecuteCommand(container, command, args);
         }
+
+        public IHttp CreateHttp()
+        {
+            var address = new Address("http://localhost", 9200);
+            var baseUrl = "";
+            return entryPoint.Tools.CreateHttp(address, baseUrl, client =>
+            {
+                // -H "kbn-xsrf: reporting" -H "Content-Type: application/json" 
+                client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
+            });
+        }
     }
 
     public interface IHasContainer

Framework/Core/Http.cs

@@ -79,7 +79,7 @@ namespace Core
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
         {
-            var response = PostJson<TRequest>(route, body);
+            var response = PostJson(route, body);
             return Time.Wait(response.Content.ReadAsStringAsync());
         }
 

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -2,6 +2,9 @@
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
+using Microsoft.IdentityModel.Abstractions;
+using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using Utils;
 
@@ -10,6 +13,131 @@ namespace Tests.BasicTests
     [TestFixture]
     public class ExampleTests : CodexDistTest
     {
+        [Test]
+        public void AAA()
+        {
+            var http = this.Ci.CreateHttp();
+
+            //var query = "{\r\n  \"query\": {\r\n    \"bool\": {\r\n      \"filter\": [\r\n        {\r\n          \"term\": {\r\n            \"container_image.keyword\": \"docker.io/codexstorage/nim-codex:sha-9d735f9-dist-tests\"\r\n          }\r\n        },\r\n        {\r\n          \"term\": {\r\n            \"pod_namespace.keyword\": \"codex-continuous-tests\"\r\n          }\r\n        },\r\n        {\r\n          \"term\": {\r\n            \"pod_name.keyword\": \"codex3-workflow3-ff476767d-98zx4\"\r\n          }\r\n        },\r\n        {\r\n          \"range\": {\r\n            \"@timestamp\": {\r\n              \"lte\": \"2023-09-25T13:02:23.559Z\",\r\n              \"gt\": \"2023-09-25T20:00:00.000Z\"\r\n            }\r\n          }\r\n        }\r\n      ]\r\n    }\r\n  }\r\n}";
+            //var query = "{  \"query\": {\"bool\": {  \"filter\": [{  \"term\": {\"container_image.keyword\": \"docker.io/codexstorage/nim-codex:sha-9d735f9-dist-tests\"  }},{  \"term\": {\"pod_namespace.keyword\": \"codex-continuous-tests\"  }},{  \"term\": {\"pod_name.keyword\": \"codex3-workflow3-ff476767d-98zx4\"  }},{  \"range\": {\"@timestamp\": {  \"lte\": \"2023-09-29T13:02:23.559Z\",  \"gt\": \"2023-09-20T20:00:00.000Z\"}  }}  ]}  }}";
+
+            
+
+            //var queryTemplate = "{ \"query\": {\"bool\": { \"filter\": [{ \"term\": {\"pod_namespace.keyword\": \"codex-continuous-tests\" }},{ \"term\": {\"pod_name.keyword\": \"bootstrap-2-599dfb4d65-v4v2b\" }},{ \"range\": {\"@timestamp\": { \"lte\": \"2023-10-03T13:02:23.559Z\", \"gt\": \"2023-09-01T20:00:00.000Z\"} }} ]} },\t\"_source\": \"message\",\t\"from\": <FROM>,\t\"size\": <SIZE>}";
+            var queryTemplate = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"2023-10-01T00:00:00.000Z\", \"lte\": \"2023-10-03T09:00:00.000Z\" } } }, { \"match_phrase\": { \"pod_name\": \"bootstrap-2-599dfb4d65-v4v2b\" } } ] } } }";
+
+            var outputFile = "c:\\Users\\Ben\\Desktop\\Cluster\\reconstructed.log";
+
+            var sizeOfPage = 2000;
+            var searchAfter = "";
+            var lastHits = 1;
+            var totalHits = 0;
+            var lastLogLine = -1;
+
+            var queue = new List<QueryLogEntry>();
+            //var jumpback = 0;
+
+            while (lastHits > 0)
+            {
+                //if (queue.Any()) jumpback++;
+                //else jumpback = 0;
+
+                var query = queryTemplate
+                                .Replace("<SIZE>", sizeOfPage.ToString())
+                                .Replace("<SEARCHAFTER>", searchAfter);
+
+                var output = http.HttpPostString("_search", query);
+
+                var response = JsonConvert.DeserializeObject<SearchResponse>(output)!;
+
+                lastHits = response.hits.hits.Length;
+                totalHits += response.hits.hits.Length;
+                if (lastHits > 0)
+                {
+                    var uniqueSearchNumbers = response.hits.hits.Select(h => h.sort.Single()).Distinct().ToList();
+                    uniqueSearchNumbers.Reverse();
+                    var searchNumber = uniqueSearchNumbers.Skip(1).First().ToString();
+                    searchAfter = $"\"search_after\": [{searchNumber}],";
+
+                    foreach (var hit in response.hits.hits)
+                    {
+                        var message = hit.fields.message.Single();
+                        var sub = message.Substring(0, 12);
+                        if (int.TryParse(sub, out int number))
+                        {
+                            queue.Add(new QueryLogEntry(message, number));
+                        }
+                    }
+                }   
+
+                // unload queue
+                var runQueue = 1;
+                while (runQueue > 0)
+                {
+                    var wantedNumber = lastLogLine + 1;
+                    var oldOnes = queue.Where(e => e.Number < wantedNumber).ToList();
+                    foreach (var old in oldOnes) queue.Remove(old);
+
+                    var entry = queue.FirstOrDefault(e => e.Number == wantedNumber);
+                    if (entry != null)
+                    {
+                        File.AppendAllLines(outputFile, new[] { entry.Message });
+                        queue.Remove(entry);
+                        lastLogLine = entry.Number;
+                        if (!queue.Any()) runQueue = 0;
+                    }
+                    else
+                    {
+                        runQueue = 0;
+                    }
+                }
+            }
+
+            
+            //var ouput = http.HttpGetString("");
+
+            var aaa = 0;
+        }
+
+        public class QueryLogEntry
+        {
+            public QueryLogEntry(string message, int number)
+            {
+                Message = message;
+                Number = number;
+            }
+
+            public string Message { get; }
+            public int Number { get; }
+
+            public override string ToString()
+            {
+                return Number.ToString();
+            }
+        }
+
+        public class SearchResponse
+        {
+            public SearchHits hits { get; set; }
+        }
+
+        public class SearchHits
+        {
+            public SearchHitEntry[] hits { get; set; }
+        }
+
+        public class SearchHitEntry
+        {
+            public SearchHitFields fields { get; set; }
+            public long[] sort { get; set; }
+        }
+
+        public class SearchHitFields
+        {
+            public string[] @timestamp { get; set; }
+            public string[] message { get; set; }
+        }
+
         [Test]
         public void CodexLogExample()
         {