status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes startup of log reconstruction

This commit is contained in:
benbierens 2023-10-03 16:11:14 +02:00
parent 696fb10386
commit 917d715497
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
1 changed files with 9 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs
View File

@ -37,7 +37,7 @@ namespace ContinuousTests
var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc); var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc);
targetFile.Write($"Downloading '{container.Name}' to '{targetFile.FullFilename}'."); targetFile.Write($"Downloading '{container.Name}' to '{targetFile.FullFilename}'.");
var reconstructor = new LogReconstructor(log, targetFile, http, queryTemplate); var reconstructor = new LogReconstructor(targetFile, http, queryTemplate);
reconstructor.DownloadFullLog(); reconstructor.DownloadFullLog();
log.Log("Log download finished."); log.Log("Log download finished.");
@ -50,13 +50,10 @@ namespace ContinuousTests
var end = endUtc.ToString("o"); var end = endUtc.ToString("o");
var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }"; var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
var result = source return source
.Replace("<STARTTIME>", start) .Replace("<STARTTIME>", start)
.Replace("<ENDTIME>", end) .Replace("<ENDTIME>", end)
.Replace("<PODNAME>", podName); .Replace("<PODNAME>", podName);
log.Log($"query template: '{result}'");
return result;
} }
private IHttp CreateElasticSearchHttp() private IHttp CreateElasticSearchHttp()
@ -66,8 +63,6 @@ namespace ContinuousTests
var address = new Address($"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200); var address = new Address($"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
var baseUrl = ""; var baseUrl = "";
log.Log("elastic search: " + address.Host + ":" + address.Port);
return tools.CreateHttp(address, baseUrl, client => return tools.CreateHttp(address, baseUrl, client =>
{ {
client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting"); client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
@ -77,18 +72,16 @@ namespace ContinuousTests
public class LogReconstructor public class LogReconstructor
{ {
private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>(); private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>();
private readonly ILog log;
private readonly LogFile targetFile; private readonly LogFile targetFile;
private readonly IHttp http; private readonly IHttp http;
private readonly string queryTemplate; private readonly string queryTemplate;
private const int sizeOfPage = 2000; private const int sizeOfPage = 2000;
private string searchAfter = ""; private string searchAfter = "";
private int lastHits = 1; private int lastHits = 1;
private ulong lastLogLine = 0; private ulong? lastLogLine;
public LogReconstructor(ILog log, LogFile targetFile, IHttp http, string queryTemplate) public LogReconstructor(LogFile targetFile, IHttp http, string queryTemplate)
{ {
this.log = log;
this.targetFile = targetFile; this.targetFile = targetFile;
this.http = http; this.http = http;
this.queryTemplate = queryTemplate; this.queryTemplate = queryTemplate;
@ -109,12 +102,8 @@ namespace ContinuousTests
.Replace("<SIZE>", sizeOfPage.ToString()) .Replace("<SIZE>", sizeOfPage.ToString())
.Replace("<SEARCHAFTER>", searchAfter); .Replace("<SEARCHAFTER>", searchAfter);
log.Log($"query with size {sizeOfPage} and searchAfter '{searchAfter}'.");
var response = http.HttpPostString<SearchResponse>("_search", query); var response = http.HttpPostString<SearchResponse>("_search", query);
log.Log("number of hits: " + response.hits.hits.Length);
lastHits = response.hits.hits.Length; lastHits = response.hits.hits.Length;
if (lastHits > 0) if (lastHits > 0)
{ {
@ -168,14 +157,14 @@ namespace ContinuousTests
private void ProcessQueue() private void ProcessQueue()
{ {
log.Log($"queue length: {queue.Count}"); if (lastLogLine == null)
log.Log("lowest number: " + queue.Min(q => q.Number)); {
log.Log("highest number: " + queue.Max(q => q.Number)); lastLogLine = queue.Min(q => q.Number) - 1;
}
while (queue.Any()) while (queue.Any())
{ {
ulong wantedNumber = lastLogLine + 1; ulong wantedNumber = lastLogLine.Value + 1;
log.Log("looking for number: " + wantedNumber);
DeleteOldEntries(wantedNumber); DeleteOldEntries(wantedNumber);