logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-06 23:43:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes elastic-search query

Browse Source
This commit is contained in:
ThatBen 2025-05-20 15:42:03 +02:00
parent 2ca2a6149c
commit 9fec8b341b
No known key found for this signature in database
GPG Key ID: E020A7DDCD52E1AB
3 changed files with 15 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Tools/TraceContract/Config.cs
View File

@ -16,8 +16,7 @@
public TimeSpan LogStartBeforeStorageContractStarts { get; } = TimeSpan.FromMinutes(1.0);
public string ElasticSearchUrl { get; } = $"https://es.testnet.codex.storage";
public string StorageNodesKubernetesNamespace = "codex";
public string[] StorageNodesKubernetesContainerNames = [
public string[] StorageNodesKubernetesPodNames = [
"codex-1-1",
"codex-2-1",
"codex-3-1",

22
Tools/TraceContract/ElasticSearchLogDownloader.cs
View File

@ -19,11 +19,11 @@ namespace TraceContract
this.config = config;
}
public void Download(LogFile targetFile, string containerName, DateTime startUtc, DateTime endUtc)
public void Download(LogFile targetFile, string podName, DateTime startUtc, DateTime endUtc)
{
try
{
DownloadLog(targetFile, containerName, startUtc, endUtc);
DownloadLog(targetFile, podName, startUtc, endUtc);
}
catch (Exception ex)
{
@ -31,22 +31,22 @@ namespace TraceContract
}
}
private void DownloadLog(LogFile targetFile, string containerName, DateTime startUtc, DateTime endUtc)
private void DownloadLog(LogFile targetFile, string podName, DateTime startUtc, DateTime endUtc)
{
log.Log($"Downloading log (from ElasticSearch) for container '{containerName}' within time range: " +
log.Log($"Downloading log (from ElasticSearch) for pod '{podName}' within time range: " +
$"{startUtc.ToString("o")} - {endUtc.ToString("o")}");
var endpoint = CreateElasticSearchEndpoint();
var queryTemplate = CreateQueryTemplate(containerName, startUtc, endUtc);
var queryTemplate = CreateQueryTemplate(podName, startUtc, endUtc);
targetFile.Write($"Downloading '{containerName}' to '{targetFile.Filename}'.");
targetFile.Write($"Downloading '{podName}' to '{targetFile.Filename}'.");
var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
reconstructor.DownloadFullLog();
log.Log("Log download finished.");
}
private string CreateQueryTemplate(string containerName, DateTime startUtc, DateTime endUtc)
private string CreateQueryTemplate(string podName, DateTime startUtc, DateTime endUtc)
{
var start = startUtc.ToString("o");
var end = endUtc.ToString("o");
@ -55,12 +55,12 @@ namespace TraceContract
// pod_namespace : codex - continuous - nolimits - tests - 1
//var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"pod_name\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"container_name\": \"<CONTAINERNAME>\" } }, { \"match_phrase\": { \"pod_namespace\": \"<NAMESPACENAME>\" } } ] } } }";
var source = "{ \"sort\": [ { \"@timestamp\": { \"order\": \"asc\" } } ], \"fields\": [ { \"field\": \"@timestamp\", \"format\": \"strict_date_optional_time\" }, { \"field\": \"message\" } ], \"size\": <SIZE>, <SEARCHAFTER> \"_source\": false, \"query\": { \"bool\": { \"must\": [], \"filter\": [ { \"range\": { \"@timestamp\": { \"format\": \"strict_date_optional_time\", \"gte\": \"<STARTTIME>\", \"lte\": \"<ENDTIME>\" } } }, { \"match_phrase\": { \"pod_name\": \"<PODNAME>\" } } ] } } }";
return source
.Replace("<STARTTIME>", start)
.Replace("<ENDTIME>", end)
.Replace("<CONTAINERNAME>", containerName)
.Replace("<NAMESPACENAME>", config.StorageNodesKubernetesNamespace);
.Replace("<PODNAME>", podName);
//.Replace("<NAMESPACENAME>", config.StorageNodesKubernetesNamespace);
}
private IEndpoint CreateElasticSearchEndpoint()
@ -121,7 +121,7 @@ namespace TraceContract
.Replace("<SIZE>", sizeOfPage.ToString())
.Replace("<SEARCHAFTER>", searchAfter);
var response = endpoint.HttpPostString<SearchResponse>("_search", query);
var response = endpoint.HttpPostString<SearchResponse>("/_search", query);
lastHits = response.hits.hits.Length;
if (lastHits > 0)

8
Tools/TraceContract/Program.cs
View File

@ -65,7 +65,6 @@ namespace TraceContract
private ICodexContracts ConnectCodexContracts(CoreInterface ci)
{
var account = EthAccountGenerator.GenerateNew();
var blockCache = new BlockCache();
var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey);
@ -83,14 +82,13 @@ namespace TraceContract
{
var start = requestTimeRange.From - config.LogStartBeforeStorageContractStarts;
foreach (var node in config.StorageNodesKubernetesContainerNames)
foreach (var node in config.StorageNodesKubernetesPodNames)
{
Log($"Downloading logs from '{node}'...");
var targetFile = output.CreateNodeLogTargetFile(node);
targetFile.Write("TODO!");
//var downloader = new ElasticSearchLogDownloader(log, tools, config);
//downloader.Download(targetFile, node, start, requestTimeRange.To);
var downloader = new ElasticSearchLogDownloader(log, tools, config);
downloader.Download(targetFile, node, start, requestTimeRange.To);
}
}