From c3ec64f25e7691f1cca0efd515ccd890ddb00d5d Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 2 Oct 2023 12:01:49 +0200 Subject: [PATCH] very very WIP for how to download complete log from elastic search --- Framework/Core/CoreInterface.cs | 12 ++ Framework/Core/Http.cs | 2 +- Tests/CodexTests/BasicTests/ExampleTests.cs | 128 ++++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/Framework/Core/CoreInterface.cs b/Framework/Core/CoreInterface.cs index 3b137d8..217641d 100644 --- a/Framework/Core/CoreInterface.cs +++ b/Framework/Core/CoreInterface.cs @@ -1,4 +1,5 @@ using KubernetesWorkflow; +using Utils; namespace Core { @@ -57,6 +58,17 @@ namespace Core var workflow = entryPoint.Tools.CreateWorkflow(); return workflow.ExecuteCommand(container, command, args); } + + public IHttp CreateHttp() + { + var address = new Address("http://localhost", 9200); + var baseUrl = ""; + return entryPoint.Tools.CreateHttp(address, baseUrl, client => + { + // -H "kbn-xsrf: reporting" -H "Content-Type: application/json" + client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting"); + }); + } } public interface IHasContainer diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 5bbd132..aef61a9 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -79,7 +79,7 @@ namespace Core public string HttpPostJson(string route, TRequest body) { - var response = PostJson(route, body); + var response = PostJson(route, body); return Time.Wait(response.Content.ReadAsStringAsync()); } diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 4049b9a..cac9935 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -2,6 +2,9 @@ using DistTestCore; using GethPlugin; using MetricsPlugin; +using Microsoft.IdentityModel.Abstractions; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; +using Newtonsoft.Json; using NUnit.Framework; using Utils; @@ -10,6 +13,131 @@ namespace Tests.BasicTests [TestFixture] public class ExampleTests : CodexDistTest { + [Test] + public void AAA() + { + var http = this.Ci.CreateHttp(); + + //var query = "{\r\n \"query\": {\r\n \"bool\": {\r\n \"filter\": [\r\n {\r\n \"term\": {\r\n \"container_image.keyword\": \"docker.io/codexstorage/nim-codex:sha-9d735f9-dist-tests\"\r\n }\r\n },\r\n {\r\n \"term\": {\r\n \"pod_namespace.keyword\": \"codex-continuous-tests\"\r\n }\r\n },\r\n {\r\n \"term\": {\r\n \"pod_name.keyword\": \"codex3-workflow3-ff476767d-98zx4\"\r\n }\r\n },\r\n {\r\n \"range\": {\r\n \"@timestamp\": {\r\n \"lte\": \"2023-09-25T13:02:23.559Z\",\r\n \"gt\": \"2023-09-25T20:00:00.000Z\"\r\n }\r\n }\r\n }\r\n ]\r\n }\r\n }\r\n}"; + //var query = "{ \"query\": {\"bool\": { \"filter\": [{ \"term\": {\"container_image.keyword\": \"docker.io/codexstorage/nim-codex:sha-9d735f9-dist-tests\" }},{ \"term\": {\"pod_namespace.keyword\": \"codex-continuous-tests\" }},{ \"term\": {\"pod_name.keyword\": \"codex3-workflow3-ff476767d-98zx4\" }},{ \"range\": {\"@timestamp\": { \"lte\": \"2023-09-29T13:02:23.559Z\", \"gt\": \"2023-09-20T20:00:00.000Z\"} }} ]} }}"; + + + + //var queryTemplate = "{ \"query\": {\"bool\": { \"filter\": [{ \"term\": {\"pod_namespace.keyword\": \"codex-continuous-tests\" }},{ \"term\": {\"pod_name.keyword\": \"bootstrap-2-599dfb4d65-v4v2b\" }},{ \"range\": {\"@timestamp\": { \"lte\": \"2023-10-03T13:02:23.559Z\", \"gt\": \"2023-09-01T20:00:00.000Z\"} }} ]} },\t\"_source\": \"message\",\t\"from\": ,\t\"size\": }"; + var queryTemplate = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": , \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"2023-10-01T00:00:00.000Z\", \"lte\": \"2023-10-03T09:00:00.000Z\" } } }, { \"match_phrase\": { \"pod_name\": \"bootstrap-2-599dfb4d65-v4v2b\" } } ] } } }"; + + var outputFile = "c:\\Users\\Ben\\Desktop\\Cluster\\reconstructed.log"; + + var sizeOfPage = 2000; + var searchAfter = ""; + var lastHits = 1; + var totalHits = 0; + var lastLogLine = -1; + + var queue = new List(); + //var jumpback = 0; + + while (lastHits > 0) + { + //if (queue.Any()) jumpback++; + //else jumpback = 0; + + var query = queryTemplate + .Replace("", sizeOfPage.ToString()) + .Replace("", searchAfter); + + var output = http.HttpPostString("_search", query); + + var response = JsonConvert.DeserializeObject(output)!; + + lastHits = response.hits.hits.Length; + totalHits += response.hits.hits.Length; + if (lastHits > 0) + { + var uniqueSearchNumbers = response.hits.hits.Select(h => h.sort.Single()).Distinct().ToList(); + uniqueSearchNumbers.Reverse(); + var searchNumber = uniqueSearchNumbers.Skip(1).First().ToString(); + searchAfter = $"\"search_after\": [{searchNumber}],"; + + foreach (var hit in response.hits.hits) + { + var message = hit.fields.message.Single(); + var sub = message.Substring(0, 12); + if (int.TryParse(sub, out int number)) + { + queue.Add(new QueryLogEntry(message, number)); + } + } + } + + // unload queue + var runQueue = 1; + while (runQueue > 0) + { + var wantedNumber = lastLogLine + 1; + var oldOnes = queue.Where(e => e.Number < wantedNumber).ToList(); + foreach (var old in oldOnes) queue.Remove(old); + + var entry = queue.FirstOrDefault(e => e.Number == wantedNumber); + if (entry != null) + { + File.AppendAllLines(outputFile, new[] { entry.Message }); + queue.Remove(entry); + lastLogLine = entry.Number; + if (!queue.Any()) runQueue = 0; + } + else + { + runQueue = 0; + } + } + } + + + //var ouput = http.HttpGetString(""); + + var aaa = 0; + } + + public class QueryLogEntry + { + public QueryLogEntry(string message, int number) + { + Message = message; + Number = number; + } + + public string Message { get; } + public int Number { get; } + + public override string ToString() + { + return Number.ToString(); + } + } + + public class SearchResponse + { + public SearchHits hits { get; set; } + } + + public class SearchHits + { + public SearchHitEntry[] hits { get; set; } + } + + public class SearchHitEntry + { + public SearchHitFields fields { get; set; } + public long[] sort { get; set; } + } + + public class SearchHitFields + { + public string[] @timestamp { get; set; } + public string[] message { get; set; } + } + [Test] public void CodexLogExample() {