From 08f42e9cbb6253a48ab113cb47ded94708cbd25f Mon Sep 17 00:00:00 2001 From: gmega Date: Tue, 14 Nov 2023 20:35:31 -0300 Subject: [PATCH] allow specifying time bound for resource listing operations, increase aggregation bucket count --- logtools/cli/es_logs.py | 6 + logtools/log/base.py | 4 +- .../elastic_search/elastic_search_log_repo.py | 33 +- .../elastic_search/elastic_search_source.py | 2 +- ...time_horizon_for_retrieving_resources.yaml | 58 ++ ...t_should_retrieve_existing_namespaces.yaml | 4 +- ..._retrieve_existing_pods_for_namespace.yaml | 4 +- .../test_should_fetch_logs_by_date.yaml | 920 +++++++++--------- .../tests/test_elasticsearch_log_repo.py | 22 +- .../sources/input/tests/test_file_source.py | 2 +- .../parse/tests/test_chronicles_raw_source.py | 2 +- 11 files changed, 576 insertions(+), 481 deletions(-) create mode 100644 logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_respect_time_horizon_for_retrieving_resources.yaml diff --git a/logtools/cli/es_logs.py b/logtools/cli/es_logs.py index 2ed0abe..c4feb05 100644 --- a/logtools/cli/es_logs.py +++ b/logtools/cli/es_logs.py @@ -1,7 +1,9 @@ +import datetime import math import os import textwrap from argparse import ArgumentParser +from datetime import timedelta from enum import Enum from typing import List, Iterable @@ -77,6 +79,10 @@ def main(): subparsers = parser.add_subparsers(title='Command', required=True) get = subparsers.add_parser('get', help='Display existing resources') + get.add_argument('--from', type=tsparser.parse, + help='Show resources present in log messages starting at the given date ' + '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.', + default=(datetime.datetime.today() - timedelta(days=7)).date()) get.set_defaults(main=get_object) get_subparsers = get.add_subparsers(title='Resource type', dest='resource_type', required=True) diff --git a/logtools/log/base.py b/logtools/log/base.py index 821c5c4..a13934a 100644 --- a/logtools/log/base.py +++ b/logtools/log/base.py @@ -1,10 +1,8 @@ import abc -import re from abc import ABC from dataclasses import dataclass from datetime import datetime -from enum import Enum -from typing import Optional, TypeVar, Generic, Iterable, Iterator +from typing import TypeVar, Generic, Iterator TLocation = TypeVar('TLocation') diff --git a/logtools/log/sources/input/elastic_search/elastic_search_log_repo.py b/logtools/log/sources/input/elastic_search/elastic_search_log_repo.py index b7a2af9..637a757 100644 --- a/logtools/log/sources/input/elastic_search/elastic_search_log_repo.py +++ b/logtools/log/sources/input/elastic_search/elastic_search_log_repo.py @@ -1,7 +1,7 @@ +import logging from dataclasses import dataclass from datetime import datetime -import logging -from typing import Optional, Iterator +from typing import Optional, Iterator, Dict, Any from elasticsearch import Elasticsearch @@ -23,11 +23,15 @@ class Pod: indices: tuple[str, ...] +MAX_AGGREGATION_BUCKETS = 1000 + + class ElasticSearchLogRepo: def __init__( self, client: Optional[Elasticsearch] = None, indices: str = 'continuous-tests-pods-*', + since: Optional[datetime] = None, ): if client is None: logger.warning('No client provided, defaulting to localhost') @@ -35,20 +39,21 @@ class ElasticSearchLogRepo: self.client = client self.indices = indices + self.since = since def namespaces(self, prefix: Optional[str] = None) -> Iterator[Namespace]: - query = { + query = self._time_limited({ 'size': 0, 'aggs': { 'distinct_namespaces': { - 'terms': {'field': 'pod_namespace.keyword'}, + 'terms': {'field': 'pod_namespace.keyword', 'size': MAX_AGGREGATION_BUCKETS}, 'aggs': { 'indices': {'terms': {'field': '_index'}}, 'runid': {'terms': {'field': 'pod_labels.runid.keyword'}}, } } } - } + }) if prefix is not None: query['aggs']['distinct_namespaces']['terms']['include'] = f'{prefix}.*' # type: ignore @@ -63,19 +68,19 @@ class ElasticSearchLogRepo: ) def pods(self, prefix: Optional[str] = None, run_id: Optional[str] = None): - query = { + query = self._time_limited({ 'size': 0, 'aggs': { 'distinct_pods': { - 'terms': {'field': 'pod_name.keyword'}, + 'terms': {'field': 'pod_name.keyword', 'size': MAX_AGGREGATION_BUCKETS}, 'aggs': { 'indices': {'terms': {'field': '_index'}}, 'namespace': {'terms': {'field': 'pod_namespace.keyword'}}, - 'runid': {'terms': {'field': 'pod_labels.runid.keyword'}}, + 'runid': {'terms': {'field': 'pod_labels.runid.keyword', 'size': MAX_AGGREGATION_BUCKETS}}, } } } - } + }) if prefix is not None: query['aggs']['distinct_pods']['terms']['include'] = f'{prefix}.*' # type: ignore @@ -98,3 +103,13 @@ class ElasticSearchLogRepo: run_id=pod['runid']['buckets'][0]['key'], indices=tuple(sorted(index['key'] for index in pod['indices']['buckets'])) ) + + def _time_limited(self, query: Dict[str, Any]) -> Dict[str, Any]: + if self.since is not None: + query['query'] = { + 'bool': { + 'filter': [{'range': {'@timestamp': {'gte': self.since.isoformat()}}}] + } + } + + return query diff --git a/logtools/log/sources/input/elastic_search/elastic_search_source.py b/logtools/log/sources/input/elastic_search/elastic_search_source.py index c9d96a0..4aa0e96 100644 --- a/logtools/log/sources/input/elastic_search/elastic_search_source.py +++ b/logtools/log/sources/input/elastic_search/elastic_search_source.py @@ -1,7 +1,7 @@ import logging from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Optional, List, Dict, Any, Iterator, Set +from typing import Optional, Dict, Any, Iterator, Set from elasticsearch import Elasticsearch diff --git a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_respect_time_horizon_for_retrieving_resources.yaml b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_respect_time_horizon_for_retrieving_resources.yaml new file mode 100644 index 0000000..70ac387 --- /dev/null +++ b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_respect_time_horizon_for_retrieving_resources.yaml @@ -0,0 +1,58 @@ +interactions: +- request: + body: '{"aggs":{"distinct_namespaces":{"terms":{"field":"pod_namespace.keyword","size":1000,"include":"codex-continuous-tests-profiling.*"},"aggs":{"indices":{"terms":{"field":"_index"}},"runid":{"terms":{"field":"pod_labels.runid.keyword"}}}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"gte":"2023-11-14T18:00:00+00:00"}}}]}},"size":0}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/continuous-tests-pods-*/_search + response: + body: + string: '{"took":2,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_namespaces":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[]}}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +- request: + body: '{"aggs":{"distinct_namespaces":{"terms":{"field":"pod_namespace.keyword","size":1000,"include":"codex-continuous-tests-profiling.*"},"aggs":{"indices":{"terms":{"field":"_index"}},"runid":{"terms":{"field":"pod_labels.runid.keyword"}}}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"gte":"2023-11-07T18:00:00+00:00"}}}]}},"size":0}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/continuous-tests-pods-*/_search + response: + body: + string: '{"took":2,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_namespaces":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-0","doc_count":32272649,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":23279858},{"key":"continuous-tests-pods-2023.11.09","doc_count":8405729},{"key":"continuous-tests-pods-2023.11.07","doc_count":573187},{"key":"continuous-tests-pods-2023.11.14","doc_count":13875}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-085853","doc_count":31604533},{"key":"20231107-074743","doc_count":517515},{"key":"20231109-043100","doc_count":77259},{"key":"20231107-065930","doc_count":50350},{"key":"20231114-051742","doc_count":7115},{"key":"20231107-064223","doc_count":5322},{"key":"20231114-045924","doc_count":3995},{"key":"20231109-055106","doc_count":3795},{"key":"20231114-051016","doc_count":2765}]}},{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":22615739,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":22415082},{"key":"continuous-tests-pods-2023.11.09","doc_count":200657}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":22615739}]}}]}}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +version: 1 diff --git a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_namespaces.yaml b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_namespaces.yaml index 2c9c526..611f7ac 100644 --- a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_namespaces.yaml +++ b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_namespaces.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"aggs":{"distinct_namespaces":{"terms":{"field":"pod_namespace.keyword","include":"codex-continuous-tests-profiling.*"},"aggs":{"indices":{"terms":{"field":"_index"}},"runid":{"terms":{"field":"pod_labels.runid.keyword"}}}}},"size":0}' + body: '{"aggs":{"distinct_namespaces":{"terms":{"field":"pod_namespace.keyword","size":1000,"include":"codex-continuous-tests-profiling.*"},"aggs":{"indices":{"terms":{"field":"_index"}},"runid":{"terms":{"field":"pod_labels.runid.keyword"}}}}},"size":0}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 @@ -16,7 +16,7 @@ interactions: uri: http://localhost:9200/continuous-tests-pods-*/_search response: body: - string: '{"took":24,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_namespaces":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-0","doc_count":32258774,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":23279858},{"key":"continuous-tests-pods-2023.11.09","doc_count":8405729},{"key":"continuous-tests-pods-2023.11.07","doc_count":573187}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-085853","doc_count":31604533},{"key":"20231107-074743","doc_count":517515},{"key":"20231109-043100","doc_count":77259},{"key":"20231107-065930","doc_count":50350},{"key":"20231107-064223","doc_count":5322},{"key":"20231109-055106","doc_count":3795}]}},{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":22615739,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":22415082},{"key":"continuous-tests-pods-2023.11.09","doc_count":200657}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":22615739}]}}]}}}' + string: '{"took":5327,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_namespaces":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-0","doc_count":32272649,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":23279858},{"key":"continuous-tests-pods-2023.11.09","doc_count":8405729},{"key":"continuous-tests-pods-2023.11.07","doc_count":573187},{"key":"continuous-tests-pods-2023.11.14","doc_count":13875}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-085853","doc_count":31604533},{"key":"20231107-074743","doc_count":517515},{"key":"20231109-043100","doc_count":77259},{"key":"20231107-065930","doc_count":50350},{"key":"20231114-051742","doc_count":7115},{"key":"20231107-064223","doc_count":5322},{"key":"20231114-045924","doc_count":3995},{"key":"20231109-055106","doc_count":3795},{"key":"20231114-051016","doc_count":2765}]}},{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":22615739,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":22415082},{"key":"continuous-tests-pods-2023.11.09","doc_count":200657}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":22615739}]}}]}}}' headers: Transfer-Encoding: - chunked diff --git a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_pods_for_namespace.yaml b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_pods_for_namespace.yaml index 44c151c..ae1a011 100644 --- a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_pods_for_namespace.yaml +++ b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_existing_pods_for_namespace.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"aggs":{"distinct_pods":{"terms":{"field":"pod_name.keyword"},"aggs":{"indices":{"terms":{"field":"_index"}},"namespace":{"terms":{"field":"pod_namespace.keyword"}},"runid":{"terms":{"field":"pod_labels.runid.keyword"}}}}},"query":{"bool":{"filter":[{"term":{"pod_labels.runid.keyword":"20231109-101554"}}]}},"size":0}' + body: '{"aggs":{"distinct_pods":{"terms":{"field":"pod_name.keyword","size":1000},"aggs":{"indices":{"terms":{"field":"_index"}},"namespace":{"terms":{"field":"pod_namespace.keyword"}},"runid":{"terms":{"field":"pod_labels.runid.keyword","size":1000}}}}},"query":{"bool":{"filter":[{"term":{"pod_labels.runid.keyword":"20231109-101554"}}]}},"size":0}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 @@ -16,7 +16,7 @@ interactions: uri: http://localhost:9200/continuous-tests-pods-*/_search response: body: - string: '{"took":17,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_pods":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"bootstrap-2-58b69484bc-88msf","doc_count":11145047,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":11060058},{"key":"continuous-tests-pods-2023.11.09","doc_count":84989}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":11145047}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":11145047}]}},{"key":"codex1-3-b558568cf-tvcsc","doc_count":11138278,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":11053946},{"key":"continuous-tests-pods-2023.11.09","doc_count":84332}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":11138278}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":11138278}]}},{"key":"geth-0-7d8bc9dd5b-8wx95","doc_count":332341,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":301052},{"key":"continuous-tests-pods-2023.11.09","doc_count":31289}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":332341}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":332341}]}},{"key":"ctnr4-d8f8d6d8-rtqrp","doc_count":60,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.09","doc_count":34},{"key":"continuous-tests-pods-2023.11.10","doc_count":26}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":60}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":60}]}},{"key":"codex-contracts-1-b98d98877-bqd5x","doc_count":13,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.09","doc_count":13}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":13}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":13}]}}]}}}' + string: '{"took":6073,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":10000,"relation":"gte"},"max_score":null,"hits":[]},"aggregations":{"distinct_pods":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"bootstrap-2-58b69484bc-88msf","doc_count":11145047,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":11060058},{"key":"continuous-tests-pods-2023.11.09","doc_count":84989}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":11145047}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":11145047}]}},{"key":"codex1-3-b558568cf-tvcsc","doc_count":11138278,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":11053946},{"key":"continuous-tests-pods-2023.11.09","doc_count":84332}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":11138278}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":11138278}]}},{"key":"geth-0-7d8bc9dd5b-8wx95","doc_count":332341,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.10","doc_count":301052},{"key":"continuous-tests-pods-2023.11.09","doc_count":31289}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":332341}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":332341}]}},{"key":"ctnr4-d8f8d6d8-rtqrp","doc_count":60,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.09","doc_count":34},{"key":"continuous-tests-pods-2023.11.10","doc_count":26}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":60}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":60}]}},{"key":"codex-contracts-1-b98d98877-bqd5x","doc_count":13,"indices":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"continuous-tests-pods-2023.11.09","doc_count":13}]},"namespace":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"codex-continuous-tests-profiling-two-client-tests-sched-0","doc_count":13}]},"runid":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"20231109-101554","doc_count":13}]}}]}}}' headers: Transfer-Encoding: - chunked diff --git a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_by_date.yaml b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_by_date.yaml index a2a4bfe..e311c1b 100644 --- a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_by_date.yaml +++ b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_by_date.yaml @@ -42,7 +42,7 @@ interactions: uri: http://localhost:9200/continuous-tests-pods-2023.11.10/_search?scroll=2m response: body: - string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":98,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"LG6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.000682976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":435,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"LG6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.000682976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:47.000+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yr/zb2rhhRPsAito8aEDtDMDY3b731ZR8hconySZrLXCQ1CJxnYr count=7615994","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287000]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LW6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.000732800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -269,13 +269,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Yr/zb2rhWiYZi4FQBokXseJnawVM52qQcuK5HjsVBcmThPgogJYr count=7616044","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287511]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X26nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.511923752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:47.511+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*gogJYr count=7616045","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287511]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.516607489Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*gogJYr count=7616045","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287511]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.516678323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:47.516+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*gogJYr inflight=0 count=7616048","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.516607489Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:47.516+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616046","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YW6nt4sBOvK4VyD1dwpr","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.516655120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:47.516+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*gogJYr inflight=1 count=7616047","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.516678323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:47.516+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*gogJYr inflight=0 count=7616048","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW6nt4sBOvK4VyD1fRFU","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.521958675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*gogJYr inflight=1 count=7616047","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593287516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW6nt4sBOvK4VyD1fRFU","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.521958675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:47.521+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Da/zb2rhoZmUgRhZ2CfRRMXXEiwbNSJVazu7rzqTRRij5UqGD6Da count=7642893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593287521]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Xm6nt4sBOvK4VyD1fRFU","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:47.522010150Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -926,17 +926,17 @@ interactions: 2023-11-10 05:14:48.999+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*BjY6Da inflight=1 count=7643041","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289000]},{"_index":"continuous-tests-pods-2023.11.10","_id":"u26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.000204478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.000+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*BjY6Da inflight=0 count=7643042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289000]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046805499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.046+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Da/zb2rhkmMG7q8xGx8TuYGZXd7ifZ8mqxJ6crjagjfLUKDAf4Da - count=7643043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046841331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.046+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*DAf4Da count=7643044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"om6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046752168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*BjY6Da inflight=0 count=7643042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289000]},{"_index":"continuous-tests-pods-2023.11.10","_id":"om6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046752168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.046+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/zS/zb2rhhefE6qqexenHzFftF9pSF3cRrTsus6FyYnZzjNjXZ6zS count=7616195","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"o26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046804242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.046+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*jXZ6zS count=7616196","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.050695781Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*jXZ6zS count=7616196","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046805499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.046+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Da/zb2rhkmMG7q8xGx8TuYGZXd7ifZ8mqxJ6crjagjfLUKDAf4Da + count=7643043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.046841331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.046+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*DAf4Da count=7643044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289046]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.050695781Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.050+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643045","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289050]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.050722313Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.050+00:00 Advertising block topics=\"codex @@ -948,17 +948,17 @@ interactions: 2023-11-10 05:14:49.052+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*jXZ6zS inflight=1 count=7616198","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289052]},{"_index":"continuous-tests-pods-2023.11.10","_id":"pm6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.052798246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.052+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*jXZ6zS inflight=0 count=7616199","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289052]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wW6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097252833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.096+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Da/zb2rhYULBu2WPfgFeX2NHUuQ8W9Y1UtQe2qVaxAe8mkLLBZDa - count=7643048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097327274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.097+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*LLBZDa count=7643049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"p26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097404913Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*jXZ6zS inflight=0 count=7616199","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289052]},{"_index":"continuous-tests-pods-2023.11.10","_id":"p26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097404913Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.096+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/zS/zb2rhg4Z4dLjBAgxBjA8CpAwauBX3Q5FL9RdNr3LufkyG9bzS count=7616200","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qG6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097460718Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.097+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*yG9bzS count=7616201","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.102598999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*yG9bzS count=7616201","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wW6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097252833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.096+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Da/zb2rhYULBu2WPfgFeX2NHUuQ8W9Y1UtQe2qVaxAe8mkLLBZDa + count=7643048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.097327274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.097+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*LLBZDa count=7643049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289097]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.102598999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.102+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643050","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289102]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xG6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.102637195Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.102+00:00 Advertising block topics=\"codex @@ -970,17 +970,17 @@ interactions: 2023-11-10 05:14:49.104+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*yG9bzS inflight=1 count=7616203","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"q26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.105539678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.105+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*yG9bzS inflight=0 count=7616204","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148375724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.148+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Da/zb2rhn1AwLBjELa3hPigK8Jwc1oAWwDzXMvCHDwqXqm9WeiDa - count=7643053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"x26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148417983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:49.148+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*9WeiDa count=7643054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rG6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148172318Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*yG9bzS inflight=0 count=7616204","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rG6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148172318Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.147+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/zS/zb2rharSqCCJwiXScxCspUjcU51jx2EQMJNqJXx16RBVcQjzS count=7616205","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rW6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148211239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.147+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*VcQjzS count=7616206","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yG6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.153074841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*VcQjzS count=7616206","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xm6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148375724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.148+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Da/zb2rhn1AwLBjELa3hPigK8Jwc1oAWwDzXMvCHDwqXqm9WeiDa + count=7643053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"x26nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.148417983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:49.148+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*9WeiDa count=7643054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289148]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yG6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.153074841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.152+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643055","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289153]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yW6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.153107939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.152+00:00 Advertising block topics=\"codex @@ -1041,16 +1041,16 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhbFSBL2d43yvZhT7HZedHybEz5zdnYTAa8Gi6SWgvxDzS count=7616220","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289301]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.301689065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.301+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*gvxDzS count=7616221","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289301]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1W6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306611383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*gvxDzS count=7616221","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289301]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306946870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:49.306+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7616222","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306983127Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:49.306+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*gvxDzS inflight=1 count=7616223","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1W6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306611383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.306+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Da/zb2rhY55iA4VPn7Vuc5xtQMp34JSKnTuQ51VvVUkVUTnDFiDa count=7643068","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1m6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306650946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.306+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*nDFiDa count=7643069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306946870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:49.306+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7616222","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.306983127Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:49.306+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*gvxDzS inflight=1 count=7616223","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.307184987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*nDFiDa count=7643069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289306]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.307184987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.306+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*gvxDzS inflight=0 count=7616224","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289307]},{"_index":"continuous-tests-pods-2023.11.10","_id":"126nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.311461051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.311+00:00 Provided to nodes topics=\"codex @@ -1094,15 +1094,15 @@ interactions: 2023-11-10 05:14:49.411+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616232","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yG6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.411851290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.411+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*7ucczS inflight=1 count=7616233","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4W6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412215036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*7ucczS inflight=1 count=7616233","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yW6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412131710Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:49.411+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7ucczS inflight=0 count=7616234","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4W6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412215036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.411+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643080","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4m6nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412403042Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.412+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*q96LDa inflight=1 count=7643081","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"426nt4sBOvK4VyD1jSEC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412509036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:49.412+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*q96LDa inflight=0 count=7643082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yW6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.412131710Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:49.411+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*7ucczS inflight=0 count=7616234","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"ym6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.454782768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*q96LDa inflight=0 count=7643082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593289412]},{"_index":"continuous-tests-pods-2023.11.10","_id":"ym6nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.454782768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:49.454+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/zS/zb2rheAwMVF4JG7EMiWKyzqytZWY1xfTe9R6tLjkUoaAAhzzS count=7616235","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593289454]},{"_index":"continuous-tests-pods-2023.11.10","_id":"y26nt4sBOvK4VyD1hxo4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:49.454853424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -1349,18 +1349,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhhbiPVeDNnHZzHm96LCvyhSc1qXKSyPECmWLVgX8xAKzS count=7616290","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290015]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Am6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.015359080Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.015+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*8xAKzS count=7616291","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290015]},{"_index":"continuous-tests-pods-2023.11.10","_id":"G26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020589587Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.020+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhiewaRPZyXUUa8kKdFwX85LjtaCmjfvXpJYMotT6iCNGs - count=7643138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020629875Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.020+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*6iCNGs count=7643139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"A26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020767276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*8xAKzS count=7616291","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290015]},{"_index":"continuous-tests-pods-2023.11.10","_id":"A26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020767276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.020+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616292","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"BG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020874137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.020+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*8xAKzS inflight=1 count=7616293","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"BW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020925071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.020+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*8xAKzS inflight=0 count=7616294","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.025290154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*8xAKzS inflight=0 count=7616294","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"G26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020589587Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.020+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhiewaRPZyXUUa8kKdFwX85LjtaCmjfvXpJYMotT6iCNGs + count=7643138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.020629875Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.020+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*6iCNGs count=7643139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290020]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.025290154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.025+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290025]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Hm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.025327377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.025+00:00 Advertising block topics=\"codex @@ -1393,16 +1393,16 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhnrckS4XDrib2JVRrp2WPiYy1qFp1bGqrxqSgCZXVrYzS count=7616300","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290117]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.117238880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.116+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*XVrYzS count=7616301","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290117]},{"_index":"continuous-tests-pods-2023.11.10","_id":"JW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122647478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*XVrYzS count=7616301","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290117]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122783569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:50.122+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7616302","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122829866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:50.122+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*XVrYzS inflight=1 count=7616303","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"JW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122647478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.122+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdKgXjNefJCw1LrGp5btUpW7ztk4AfANH7D1bh5mwoJGs count=7643148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Jm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122688697Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.122+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*mwoJGs count=7643149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122783569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:50.122+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7616302","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.122829866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:50.122+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*XVrYzS inflight=1 count=7616303","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.123084740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*mwoJGs count=7643149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290122]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.123084740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.122+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*XVrYzS inflight=0 count=7616304","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290123]},{"_index":"continuous-tests-pods-2023.11.10","_id":"J26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.126822878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.126+00:00 Provided to nodes topics=\"codex @@ -1437,18 +1437,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhjpfEgjQTxmSTn6ZknBoGUW59vUpG2t5LdrLxZpLaHTzS count=7616310","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290218]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.218968939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.218+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*LaHTzS count=7616311","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290218]},{"_index":"continuous-tests-pods-2023.11.10","_id":"L26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224759996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.224+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhWz8Cjmgunt9hmDpuJpc8kTd9aeYJsCm7dPZsm7rMqiGs - count=7643158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224806174Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.224+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*rMqiGs count=7643159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"F26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224163947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*LaHTzS count=7616311","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290218]},{"_index":"continuous-tests-pods-2023.11.10","_id":"F26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224163947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.223+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616312","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224212515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.223+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*LaHTzS inflight=1 count=7616313","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224247932Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.223+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*LaHTzS inflight=0 count=7616314","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.230236897Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*LaHTzS inflight=0 count=7616314","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"L26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224759996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.224+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhWz8Cjmgunt9hmDpuJpc8kTd9aeYJsCm7dPZsm7rMqiGs + count=7643158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.224806174Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.224+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*rMqiGs count=7643159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290224]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.230236897Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.229+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290230]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Mm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.230269387Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.230+00:00 Advertising block topics=\"codex @@ -1459,18 +1459,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhj5i3cDh2PECKZcTSZvPrEFphZLKKz7ABG6sWrvLxxRzS count=7616315","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290269]},{"_index":"continuous-tests-pods-2023.11.10","_id":"G26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.270038193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.269+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*LxxRzS count=7616316","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290270]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275855145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.275+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhbzNcLmLy6f4kYMzahZUs6eBrvcEkdNJgKXD1nk2bjwGs - count=7643163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275894986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.275+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*2bjwGs count=7643164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275693132Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*LxxRzS count=7616316","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290270]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275693132Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.275+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616317","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275737680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.275+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*LxxRzS inflight=1 count=7616318","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Hm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275834163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.275+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*LxxRzS inflight=0 count=7616319","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.280192629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*LxxRzS inflight=0 count=7616319","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275855145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.275+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhbzNcLmLy6f4kYMzahZUs6eBrvcEkdNJgKXD1nk2bjwGs + count=7643163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.275894986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.275+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*2bjwGs count=7643164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290275]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.280192629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.279+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290280]},{"_index":"continuous-tests-pods-2023.11.10","_id":"N26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.280225766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.280+00:00 Advertising block topics=\"codex @@ -1503,14 +1503,14 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhjDS8vCE6LCwkDhrTg1TxQrjC2YVhokZifGjp7KMKEpzS count=7616325","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290372]},{"_index":"continuous-tests-pods-2023.11.10","_id":"JW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.372107506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.371+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*MKEpzS count=7616326","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290372]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.377536275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*MKEpzS count=7616326","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290372]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Jm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.377948237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:50.377+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7616327","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290377]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.377536275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.377+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhc6RL1e6ADSeb8S7xDK4buRyKm5oMKayjfBV1DN17KkGs count=7643173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290377]},{"_index":"continuous-tests-pods-2023.11.10","_id":"P26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.377573550Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.377+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*17KkGs count=7643174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290377]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Jm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.377948237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:50.377+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7616327","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290377]},{"_index":"continuous-tests-pods-2023.11.10","_id":"J26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.378071381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*17KkGs count=7643174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290377]},{"_index":"continuous-tests-pods-2023.11.10","_id":"J26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.378071381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.377+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*MKEpzS inflight=1 count=7616328","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290378]},{"_index":"continuous-tests-pods-2023.11.10","_id":"KG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.378162146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.378+00:00 Advertised block topics=\"codex @@ -1525,18 +1525,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhfCST25Y9GTRnT6ERMHHkYGjgu5CoENXxkynWWDzZoMzS count=7616330","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290422]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Km6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.422780151Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.422+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*zZoMzS count=7616331","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290422]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428339331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.428+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhooEBiN6VvwoNYsAcsbVEqUUSFpfijQ4CYVifyxtzqtGs - count=7643178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428381513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.428+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*tzqtGs count=7643179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"K26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428573481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*zZoMzS count=7616331","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290422]},{"_index":"continuous-tests-pods-2023.11.10","_id":"K26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428573481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.428+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616332","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428612005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.428+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*zZoMzS inflight=1 count=7616333","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428787918Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.428+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*zZoMzS inflight=0 count=7616334","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.433896371Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*zZoMzS inflight=0 count=7616334","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428339331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.428+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhooEBiN6VvwoNYsAcsbVEqUUSFpfijQ4CYVifyxtzqtGs + count=7643178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.428381513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.428+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*tzqtGs count=7643179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290428]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.433896371Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.433+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290433]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.433933565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.433+00:00 Advertising block topics=\"codex @@ -1549,18 +1549,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhexU86X7iuK8ubUhoJtygoMkH6hEot52AGP9h5rtvyHzS count=7616335","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290473]},{"_index":"continuous-tests-pods-2023.11.10","_id":"L26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.473498284Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.473+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*tvyHzS count=7616336","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290473]},{"_index":"continuous-tests-pods-2023.11.10","_id":"SW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479785629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.479+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdpr8VX7yk9pLdtgjFmzqDxXE7WxpuLa2wMHfsXvynBGs - count=7643184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Sm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479832969Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.479+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*vynBGs count=7643185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479234359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*tvyHzS count=7616336","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290473]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479234359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.478+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616337","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"MW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479267759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.478+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*tvyHzS inflight=1 count=7616338","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Mm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479305527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.479+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*tvyHzS inflight=0 count=7616339","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"S26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.483525752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*tvyHzS inflight=0 count=7616339","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"SW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479785629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.479+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdpr8VX7yk9pLdtgjFmzqDxXE7WxpuLa2wMHfsXvynBGs + count=7643184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Sm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.479832969Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.479+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*vynBGs count=7643185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290479]},{"_index":"continuous-tests-pods-2023.11.10","_id":"S26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.483525752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.483+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290483]},{"_index":"continuous-tests-pods-2023.11.10","_id":"TG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.483797492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.483+00:00 Advertising block topics=\"codex @@ -1571,18 +1571,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhdYQnjZpsLcwErWHMo7webRH9dPW7hCBuLdrNbVxrhFzS count=7616340","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290524]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.524998980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.524+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*xrhFzS count=7616341","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290524]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Tm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530369924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.530+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhX1GbgSdux96b7QoVHFzi9nG8y5geatNX4yrNDjFUKqGs - count=7643189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"T26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530527381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.530+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*FUKqGs count=7643190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530823672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*xrhFzS count=7616341","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290524]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530823672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.530+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616342","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530866620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.530+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*xrhFzS inflight=1 count=7616343","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"N26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530936992Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.530+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*xrhFzS inflight=0 count=7616344","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"UG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.535455693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*xrhFzS inflight=0 count=7616344","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Tm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530369924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.530+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhX1GbgSdux96b7QoVHFzi9nG8y5geatNX4yrNDjFUKqGs + count=7643189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"T26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.530527381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.530+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*FUKqGs count=7643190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290530]},{"_index":"continuous-tests-pods-2023.11.10","_id":"UG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.535455693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.535+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290535]},{"_index":"continuous-tests-pods-2023.11.10","_id":"UW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.535491667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.535+00:00 Advertising block topics=\"codex @@ -1615,18 +1615,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhWwQnPDYWFgWoL4q5iW3vcci8mBW2H6PwdcRzjWa6qEzS count=7616350","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290626]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.626735193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.626+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*a6qEzS count=7616351","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290626]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632641921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.632+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkLRFBaXorgTSHVsMkNgVLTEw9x4Zr5LMj8JZk5CXSeGs - count=7643199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632672966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.632+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*CXSeGs count=7643200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"P26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632060114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*a6qEzS count=7616351","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290626]},{"_index":"continuous-tests-pods-2023.11.10","_id":"P26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632060114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.631+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616352","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"QG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632121359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.631+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*a6qEzS inflight=1 count=7616353","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"QW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632138327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.631+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*a6qEzS inflight=0 count=7616354","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.637969924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*a6qEzS inflight=0 count=7616354","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632641921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.632+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkLRFBaXorgTSHVsMkNgVLTEw9x4Zr5LMj8JZk5CXSeGs + count=7643199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.632672966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.632+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*CXSeGs count=7643200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290632]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.637969924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.637+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290637]},{"_index":"continuous-tests-pods-2023.11.10","_id":"W26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.638187667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.637+00:00 Advertising block topics=\"codex @@ -1637,18 +1637,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rhixx9NbRbL3qgkpREX14CNdjs8QKMpjjvraBZyr1WntzS count=7616355","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290677]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.677862262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.677+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*1WntzS count=7616356","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290677]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683758603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.683+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhf39qMgAhHWe7DAoowXgPPvKaud5wmzJNi6cbdTVCd4Gs - count=7643204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Xm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683916790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:50.683+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*VCd4Gs count=7643205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683313124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*1WntzS count=7616356","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290677]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683313124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.683+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616357","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683435727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.683+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*1WntzS inflight=1 count=7616358","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683800009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:50.683+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*1WntzS inflight=0 count=7616359","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.687934081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*1WntzS inflight=0 count=7616359","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683758603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.683+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhf39qMgAhHWe7DAoowXgPPvKaud5wmzJNi6cbdTVCd4Gs + count=7643204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Xm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.683916790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:50.683+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*VCd4Gs count=7643205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290683]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.687934081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.687+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593290687]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:50.687967677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:50.687+00:00 Advertising block topics=\"codex @@ -1793,14 +1793,14 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zS/zb2rheKqe6hXpBMoTdUJPMDqEG1iL3AACbw6vVckDcPEXhPzS count=7616391","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291035]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Z26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.035406288Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.035+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*EXhPzS count=7616392","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291035]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.039454520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*EXhPzS count=7616392","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291035]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.039990975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:51.039+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7616393","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291039]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.039454520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.039+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhj5Dbe7EszRNHZFUse6aCENJBXMaYGrnmRKBoAftQrrGs count=7643239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291039]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.039518974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.039+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*tQrrGs count=7643240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291039]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.039990975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:51.039+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7616393","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291039]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.040103289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*tQrrGs count=7643240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291039]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.040103289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.039+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*EXhPzS inflight=1 count=7616394","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291040]},{"_index":"continuous-tests-pods-2023.11.10","_id":"am6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.040336347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.040+00:00 Advertised block topics=\"codex @@ -1886,19 +1886,19 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhgvzquBcJSRHjsqaaCYVNrNoCUQUnt7e4aGgDVtFLpzGs count=7643259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291241]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.241707317Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.241+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*FLpzGs count=7643260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291241]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246666748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.246+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"l26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246711282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.246+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*FLpzGs inflight=1 count=7643262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246715974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.246+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*FLpzGs inflight=0 count=7643263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246620955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*FLpzGs count=7643260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291241]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246620955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.246+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616413","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246658397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.246+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*2qvNYu inflight=1 count=7616414","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246863102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.246+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*2qvNYu inflight=0 count=7616415","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"f26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.290341262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*2qvNYu inflight=0 count=7616415","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246666748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.246+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"l26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246711282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.246+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*FLpzGs inflight=1 count=7643262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.246715974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.246+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*FLpzGs inflight=0 count=7643263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291246]},{"_index":"continuous-tests-pods-2023.11.10","_id":"f26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.290341262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.290+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhZ7ry8zgFdCWbFHovbKpCqQwXzLfgGZs5JRkFZcRjKNYu count=7616416","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291290]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.290510631Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -1930,19 +1930,19 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhgH5jmbESnNfoqut5n7Nm85SbHWA4q4cQ3bSEUfZNuuGs count=7643269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291343]},{"_index":"continuous-tests-pods-2023.11.10","_id":"n26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.343528981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.343+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*ZNuuGs count=7643270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291343]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348224123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.347+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348265060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.348+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*ZNuuGs inflight=1 count=7643272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"om6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348457424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.348+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*ZNuuGs inflight=0 count=7643273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"hm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348517545Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*ZNuuGs count=7643270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291343]},{"_index":"continuous-tests-pods-2023.11.10","_id":"hm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348517545Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.348+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616423","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"h26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348549682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.348+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*EerkYu inflight=1 count=7616424","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"iG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348859203Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.348+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*EerkYu inflight=0 count=7616425","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"iW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.392354319Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*EerkYu inflight=0 count=7616425","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348224123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.347+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348265060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.348+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ZNuuGs inflight=1 count=7643272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"om6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.348457424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.348+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ZNuuGs inflight=0 count=7643273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291348]},{"_index":"continuous-tests-pods-2023.11.10","_id":"iW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.392354319Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.391+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rheK1FvBSyrUMyJtTY34VdgPzQP3mx9wssxSEaCm5qLeYu count=7616426","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291392]},{"_index":"continuous-tests-pods-2023.11.10","_id":"im6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.392393570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -1974,17 +1974,17 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhXdae5Psdq4e1b92wfSx2RVYK6RMKioT8xGy8du4h23Gs count=7643279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291444]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.445141669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.444+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*4h23Gs count=7643280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450903809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.450+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"q26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450963235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.450+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*4h23Gs inflight=1 count=7643282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450711114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*4h23Gs count=7643280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450711114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.450+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616433","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450755448Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.450+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*r4HKYu inflight=1 count=7616434","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"km6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450760461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.450+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*r4HKYu inflight=0 count=7616435","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.451339675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*r4HKYu inflight=0 count=7616435","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450903809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.450+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"q26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.450963235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.450+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*4h23Gs inflight=1 count=7643282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291450]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.451339675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.450+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*4h23Gs inflight=0 count=7643283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"k26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.464171226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.463+00:00 About to sleep discovery loop topics=\"codex @@ -1998,19 +1998,19 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhf9XCnhTjyWs7E23ssAycgoysnGSZ1PM6psUsP2vnjxGs count=7643284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291495]},{"_index":"continuous-tests-pods-2023.11.10","_id":"rm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.495837279Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.495+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*vnjxGs count=7643285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291495]},{"_index":"continuous-tests-pods-2023.11.10","_id":"r26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499816890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.499+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"sG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499846654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.499+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*vnjxGs inflight=1 count=7643287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"sW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499851329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.499+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*vnjxGs inflight=0 count=7643288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499793254Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*vnjxGs count=7643285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291495]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mG6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499793254Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.499+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*PU5MYu inflight=0 count=7616441","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lm6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499740513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.499+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616439","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"l26nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499788067Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.499+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*PU5MYu inflight=1 count=7616440","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.545092221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*PU5MYu inflight=1 count=7616440","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"r26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499816890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.499+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"sG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499846654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.499+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vnjxGs inflight=1 count=7643287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"sW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.499851329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.499+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vnjxGs inflight=0 count=7643288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291499]},{"_index":"continuous-tests-pods-2023.11.10","_id":"mW6nt4sBOvK4VyD1hxs4","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.545092221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.544+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhgk2Dr1fYU2cyL4nDh2LZk3C19PjuzA1fdB9zRvGJX7Yu count=7616442","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291545]},{"_index":"continuous-tests-pods-2023.11.10","_id":"M26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.545531861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -2032,83 +2032,83 @@ interactions: 2023-11-10 05:14:51.551+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*EeubGs inflight=1 count=7643292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291551]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.551521747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.551+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*EeubGs inflight=0 count=7643293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291551]},{"_index":"continuous-tests-pods-2023.11.10","_id":"t26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.596980085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.596+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhmrEwVdxvcTc4pmWvL9nrGsyWwjSvmAeVQTe3FCFDeEGs - count=7643294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291596]},{"_index":"continuous-tests-pods-2023.11.10","_id":"N26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.596276486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*EeubGs inflight=0 count=7643293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291551]},{"_index":"continuous-tests-pods-2023.11.10","_id":"N26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.596276486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.595+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhf2DRgvzHddFmc4N46Sd4aRVWLV6udciFwVYPL5HKkiYu count=7616447","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291596]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.596508189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.596+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*HKkiYu count=7616448","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291596]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.597078999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*HKkiYu count=7616448","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291596]},{"_index":"continuous-tests-pods-2023.11.10","_id":"t26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.596980085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.596+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhmrEwVdxvcTc4pmWvL9nrGsyWwjSvmAeVQTe3FCFDeEGs + count=7643294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291596]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.597078999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.596+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*FDeEGs count=7643295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291597]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602030632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*FDeEGs count=7643295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291597]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602947102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:51.602+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7616449","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Om6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602994348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:51.602+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HKkiYu inflight=1 count=7616450","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602030632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.601+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"um6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602066880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.601+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*FDeEGs inflight=1 count=7643297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"u26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602119414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.601+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*FDeEGs inflight=0 count=7643298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602947102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:51.602+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7616449","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Om6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.602994348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:51.602+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*HKkiYu inflight=1 count=7616450","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"O26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.603002702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*FDeEGs inflight=0 count=7643298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291602]},{"_index":"continuous-tests-pods-2023.11.10","_id":"O26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.603002702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.602+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*HKkiYu inflight=0 count=7616451","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291603]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647642642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.647+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhiMXgrBpsLhBp3aurFfYy8xFMbuq51pRssveKnjVjTKGs - count=7643299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647679729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.647+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*VjTKGs count=7643300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647533812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*HKkiYu inflight=0 count=7616451","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291603]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647533812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.647+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhdmdDRi6otkQEgQ4VjFadkMzCMBzGymJo4XWd5riNuhYu count=7616452","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647598509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.647+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*iNuhYu count=7616453","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651799889Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.651+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651843848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.651+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*VjTKGs inflight=1 count=7643302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651849809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.651+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*VjTKGs inflight=0 count=7643303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651891446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*iNuhYu count=7616453","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647642642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.647+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhiMXgrBpsLhBp3aurFfYy8xFMbuq51pRssveKnjVjTKGs + count=7643299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.647679729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.647+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*VjTKGs count=7643300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291647]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651891446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.651+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616454","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"P26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651950082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.651+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*iNuhYu inflight=1 count=7616455","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"QG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651963016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.651+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*iNuhYu inflight=0 count=7616456","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698304363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.698+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkTXgunW7yc1WDCV618ZWghzofYmbuijpnE7RiTCByuGs - count=7643304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698339832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.698+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*CByuGs count=7643305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"QW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698580350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*iNuhYu inflight=0 count=7616456","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651799889Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.651+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651843848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.651+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*VjTKGs inflight=1 count=7643302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.651849809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.651+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*VjTKGs inflight=0 count=7643303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"QW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698580350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.698+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhdkvznKFeFZFL14tqBt1YG3sePn2yhTVtuiA34bNUyoYu count=7616457","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Qm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698709214Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.698+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*NUyoYu count=7616458","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703295888Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.702+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703325270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.703+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CByuGs inflight=1 count=7643307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703534976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.703+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CByuGs inflight=0 count=7643308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703251890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*NUyoYu count=7616458","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698304363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.698+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkTXgunW7yc1WDCV618ZWghzofYmbuijpnE7RiTCByuGs + count=7643304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.698339832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.698+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*CByuGs count=7643305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291698]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703251890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.702+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616459","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703337535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.703+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*NUyoYu inflight=1 count=7616460","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703390873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.703+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*NUyoYu inflight=0 count=7616461","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.748925976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*NUyoYu inflight=0 count=7616461","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703295888Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.702+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703325270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.703+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*CByuGs inflight=1 count=7643307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.703534976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.703+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*CByuGs inflight=0 count=7643308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291703]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.748925976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.748+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdkkYwgZ1jX3kPzESL5x8HUYKcFvejBC7wFLjKS9rYwGs - count=7643309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291748]},{"_index":"continuous-tests-pods-2023.11.10","_id":"x26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.749009313Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.748+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*9rYwGs count=7643310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291749]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.749078947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + count=7643309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291748]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.749078947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.748+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhf3o4NmBWuMAcK2iBFagLSKwXssw4MzRCcZs3wJMdhkYu count=7616462","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291749]},{"_index":"continuous-tests-pods-2023.11.10","_id":"R26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.749328196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.748+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*MdhkYu count=7616463","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291749]},{"_index":"continuous-tests-pods-2023.11.10","_id":"SG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.753894558Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*MdhkYu count=7616463","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291749]},{"_index":"continuous-tests-pods-2023.11.10","_id":"x26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.749009313Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.748+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*9rYwGs count=7643310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291749]},{"_index":"continuous-tests-pods-2023.11.10","_id":"SG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.753894558Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.753+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616464","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291753]},{"_index":"continuous-tests-pods-2023.11.10","_id":"SW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.753945966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.753+00:00 Advertising block topics=\"codex @@ -2120,17 +2120,17 @@ interactions: 2023-11-10 05:14:51.753+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*9rYwGs inflight=1 count=7643312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291754]},{"_index":"continuous-tests-pods-2023.11.10","_id":"ym6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.754552617Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.754+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*9rYwGs inflight=0 count=7643313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291754]},{"_index":"continuous-tests-pods-2023.11.10","_id":"y26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800155703Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.799+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhXF3M6t4285RwRvYSma6bhAbFa3VSyDmGyYCe7avgs2Gs - count=7643314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800195270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.799+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*vgs2Gs count=7643315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"S26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800547028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*9rYwGs inflight=0 count=7643313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291754]},{"_index":"continuous-tests-pods-2023.11.10","_id":"S26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800547028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.799+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhhQBUgnaU522r1KjX1vvxehE7PoCDsM1cUaG9pLELQQYu count=7616467","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"TG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800574442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.799+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*ELQQYu count=7616468","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.805261090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*ELQQYu count=7616468","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"y26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800155703Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.799+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhXF3M6t4285RwRvYSma6bhAbFa3VSyDmGyYCe7avgs2Gs + count=7643314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zG6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.800195270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.799+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*vgs2Gs count=7643315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291800]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zW6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.805261090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.804+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291805]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zm6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.805297206Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.805+00:00 Advertising block topics=\"codex @@ -2152,19 +2152,19 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoQRsUo67kHs5ZkdaDuq2BNRSRc7gQB6pUwex5vYFUbGs count=7643319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291851]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.851105094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.850+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*YFUbGs count=7643320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291851]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855478227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.855+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"026nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855512167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.855+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*YFUbGs inflight=1 count=7643322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855614324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.855+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*YFUbGs inflight=0 count=7643323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Um6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855869341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*YFUbGs count=7643320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291851]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Um6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855869341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.855+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616474","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"U26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855922806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.855+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*CzjUYu inflight=1 count=7616475","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"VG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855928664Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.855+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CzjUYu inflight=0 count=7616476","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"VW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.901365075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*CzjUYu inflight=0 count=7616476","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855478227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.855+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"026nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855512167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.855+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*YFUbGs inflight=1 count=7643322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.855614324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.855+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*YFUbGs inflight=0 count=7643323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291855]},{"_index":"continuous-tests-pods-2023.11.10","_id":"VW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.901365075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.900+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhYYJGGydoHVAJikyfp3j7LVfTi8NoxbgfYuSEEN9CzfYu count=7616477","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291901]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Vm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.901433262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -2178,25 +2178,25 @@ interactions: 2023-11-10 05:14:51.906+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616479","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291906]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.906809191Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.906+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*9CzfYu inflight=1 count=7616480","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291906]},{"_index":"continuous-tests-pods-2023.11.10","_id":"126nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907239339Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*9CzfYu inflight=1 count=7616480","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291906]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907059918Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:51.906+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*9CzfYu inflight=0 count=7616481","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"126nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907239339Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.906+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907292668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.907+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*frAZGs inflight=1 count=7643327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907541097Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.907+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*frAZGs inflight=0 count=7643328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.907059918Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:51.906+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*9CzfYu inflight=0 count=7616481","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952920548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.952+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdyUHru2RhUbaGS2325Coa1Pbvskn1jgmxuYdzKQnZ4Gs - count=7643329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"226nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952956796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:51.952+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*QnZ4Gs count=7643330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952690467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*frAZGs inflight=0 count=7643328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291907]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952690467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.952+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhfjDDzmEtCsg1Yg9kxcidWc74sLsFZkx2gJ7h2BfgJkYu count=7616482","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"W26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952732607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.952+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*fgJkYu count=7616483","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"3G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.956722015Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*fgJkYu count=7616483","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952920548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.952+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhdyUHru2RhUbaGS2325Coa1Pbvskn1jgmxuYdzKQnZ4Gs + count=7643329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"226nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.952956796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:51.952+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*QnZ4Gs count=7643330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291952]},{"_index":"continuous-tests-pods-2023.11.10","_id":"3G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.956722015Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.956+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593291956]},{"_index":"continuous-tests-pods-2023.11.10","_id":"3W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.956766422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:51.956+00:00 Advertising block topics=\"codex @@ -2208,17 +2208,17 @@ interactions: 2023-11-10 05:14:51.958+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*fgJkYu inflight=1 count=7616485","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291958]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Xm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:51.959190723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:51.958+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*fgJkYu inflight=0 count=7616486","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291959]},{"_index":"continuous-tests-pods-2023.11.10","_id":"326nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003225170Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.002+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoWmtfErcKw1agE1v8pNqUYZyKdJZ22Gc5yWm6fNan7Gs - count=7643334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003262544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.003+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*Nan7Gs count=7643335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003767077Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*fgJkYu inflight=0 count=7616486","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593291959]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003767077Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.003+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhdFVrkTwJZDjByRSQgD34fWmH7oTTbYGWyT2kcXK88JYu count=7616487","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003816253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.003+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*K88JYu count=7616488","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.007077886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*K88JYu count=7616488","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"326nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003225170Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.002+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoWmtfErcKw1agE1v8pNqUYZyKdJZ22Gc5yWm6fNan7Gs + count=7643334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.003262544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.003+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*Nan7Gs count=7643335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292003]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.007077886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.006+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292007]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.007111333Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.006+00:00 Advertising block topics=\"codex @@ -2252,25 +2252,25 @@ interactions: 2023-11-10 05:14:52.058+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*NML1Yu inflight=1 count=7616495","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292059]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.059348987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.058+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*NML1Yu inflight=0 count=7616496","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292059]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105090814Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.104+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhX7GEzLwUazoJRJAbBAfeNpmHwQzDyPEvZRVwTdSzL4Gs - count=7643344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105392046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.105+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*SzL4Gs count=7643345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105820804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*NML1Yu inflight=0 count=7616496","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292059]},{"_index":"continuous-tests-pods-2023.11.10","_id":"aW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105820804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.105+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhYcxqYxB13ibSHxPgkfJT6dJdJYhi86gDXx15pck99SYu count=7616497","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"am6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105875256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.105+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*k99SYu count=7616498","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"626nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110828240Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.110+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"a26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110415641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*k99SYu count=7616498","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105090814Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.104+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhX7GEzLwUazoJRJAbBAfeNpmHwQzDyPEvZRVwTdSzL4Gs + count=7643344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.105392046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.105+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*SzL4Gs count=7643345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292105]},{"_index":"continuous-tests-pods-2023.11.10","_id":"a26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110415641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.110+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616499","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"bG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110455747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.110+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*k99SYu inflight=1 count=7616500","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"bW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110551322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.110+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*k99SYu inflight=0 count=7616501","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"7G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.111179813Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*k99SYu inflight=0 count=7616501","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"626nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.110828240Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.110+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292110]},{"_index":"continuous-tests-pods-2023.11.10","_id":"7G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.111179813Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.110+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*SzL4Gs inflight=1 count=7643347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292111]},{"_index":"continuous-tests-pods-2023.11.10","_id":"7W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.111218662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.111+00:00 Advertised block topics=\"codex @@ -2296,17 +2296,17 @@ interactions: 2023-11-10 05:14:52.165+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*DK3BGs inflight=1 count=7643352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292166]},{"_index":"continuous-tests-pods-2023.11.10","_id":"8m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.166102487Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.165+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*DK3BGs inflight=0 count=7643353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292166]},{"_index":"continuous-tests-pods-2023.11.10","_id":"826nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207719860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.207+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhjggfnkojKhoNjTMCJRFuLFt9bvZyUqbiBbnJNEcdXhGs - count=7643354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207759853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.207+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*cdXhGs count=7643355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"c26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207499894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*DK3BGs inflight=0 count=7643353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292166]},{"_index":"continuous-tests-pods-2023.11.10","_id":"c26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207499894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.207+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhXo71VjmcV7ZrbvdNgN6eRkF5z2yKznK1PxYNQV8UoRYu count=7616507","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"dG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207568762Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.207+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*8UoRYu count=7616508","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"dW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.211672593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*8UoRYu count=7616508","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"826nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207719860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.207+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhjggfnkojKhoNjTMCJRFuLFt9bvZyUqbiBbnJNEcdXhGs + count=7643354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.207759853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.207+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*cdXhGs count=7643355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292207]},{"_index":"continuous-tests-pods-2023.11.10","_id":"dW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.211672593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.211+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616509","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292211]},{"_index":"continuous-tests-pods-2023.11.10","_id":"dm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.211710321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.211+00:00 Advertising block topics=\"codex @@ -2318,17 +2318,17 @@ interactions: 2023-11-10 05:14:52.211+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*cdXhGs inflight=1 count=7643357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292212]},{"_index":"continuous-tests-pods-2023.11.10","_id":"926nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.212420067Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.212+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*cdXhGs inflight=0 count=7643358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292212]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258645018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.258+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhb27QQKenKjWvnrcVjdpwyg69DrdeRZFV26Scni3EijGs - count=7643359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258740040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.258+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*3EijGs count=7643360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"eG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258685446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*cdXhGs inflight=0 count=7643358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292212]},{"_index":"continuous-tests-pods-2023.11.10","_id":"eG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258685446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.258+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhZeXJ5xxqYsDsuVnUqi1yoZgofVVgaJUhVzSWa2jaEnYu count=7616512","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"eW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258750300Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.258+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*jaEnYu count=7616513","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.263100936Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*jaEnYu count=7616513","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-G6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258645018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.258+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhb27QQKenKjWvnrcVjdpwyg69DrdeRZFV26Scni3EijGs + count=7643359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.258740040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.258+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*3EijGs count=7643360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292258]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.263100936Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.262+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292263]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-26nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.263146470Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.262+00:00 Advertising block topics=\"codex @@ -2340,17 +2340,17 @@ interactions: 2023-11-10 05:14:52.264+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*jaEnYu inflight=1 count=7616515","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292264]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.264631564Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.264+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*jaEnYu inflight=0 count=7616516","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292264]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309771714Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.309+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rheSsginJhP6ovnpWWZ73yJLqGU8h6h3D9VmX13maDYUGs - count=7643364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309821068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.309+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*aDYUGs count=7643365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309257135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*jaEnYu inflight=0 count=7616516","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292264]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309257135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.308+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhdfUJby8GonA1a8HMP1SJQ5C6cpyk5uLTQxHEJuyithYu count=7616517","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309432974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.309+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*yithYu count=7616518","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"f26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.315451354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*yithYu count=7616518","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309771714Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.309+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rheSsginJhP6ovnpWWZ73yJLqGU8h6h3D9VmX13maDYUGs + count=7643364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_m6nt4sBOvK4VyD1jSIC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.309821068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.309+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*aDYUGs count=7643365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292309]},{"_index":"continuous-tests-pods-2023.11.10","_id":"f26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.315451354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.315+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616519","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292315]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.315731298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.315+00:00 Advertising block topics=\"codex @@ -2362,17 +2362,17 @@ interactions: 2023-11-10 05:14:52.315+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*aDYUGs inflight=1 count=7643367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292316]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.316166528Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.315+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*aDYUGs inflight=0 count=7643368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292316]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Am6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360585271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.360+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkLcMxBMDHdwXGxxTbP8nWLEQe5bJi8vhu7cqfpUGojGs - count=7643369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"A26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360623757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.360+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*UGojGs count=7643370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360366194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*aDYUGs inflight=0 count=7643368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292316]},{"_index":"continuous-tests-pods-2023.11.10","_id":"gm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360366194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.360+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhgyXMW4JNRQyJgth3vQBtsZwdsHPaheHTyFyHiQ67oTYu count=7616522","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"g26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360554160Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.360+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*67oTYu count=7616523","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"BG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.365473788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*67oTYu count=7616523","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Am6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360585271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.360+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhkLcMxBMDHdwXGxxTbP8nWLEQe5bJi8vhu7cqfpUGojGs + count=7643369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"A26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.360623757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.360+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*UGojGs count=7643370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292360]},{"_index":"continuous-tests-pods-2023.11.10","_id":"BG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.365473788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.365+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292365]},{"_index":"continuous-tests-pods-2023.11.10","_id":"BW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.365510529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.365+00:00 Advertising block topics=\"codex @@ -2394,51 +2394,51 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoKmUyaR7UnbRUknHViAd7nsoMPiYj6F8SfDaBQVZbHGs count=7643374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.411133197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.410+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*VZbHGs count=7643375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415727431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.415+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Cm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415778411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.415+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*VZbHGs inflight=1 count=7643377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"C26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415896287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.415+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*VZbHGs inflight=0 count=7643378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"iW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415090981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*VZbHGs count=7643375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"iW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415090981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.414+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616529","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"im6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415119104Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.414+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*97VmYu inflight=1 count=7616530","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"i26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415326260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.415+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*97VmYu inflight=0 count=7616531","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.461863484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*97VmYu inflight=0 count=7616531","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415727431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.415+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Cm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415778411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.415+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*VZbHGs inflight=1 count=7643377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"C26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.415896287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.415+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*VZbHGs inflight=0 count=7643378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292415]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.461863484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.461+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhooBhFMraNXx4GvzTM9PknbhDSZczUo2MRmfWmkmQyPYu - count=7616532","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292461]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.462410478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + count=7616532","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292461]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.462062563Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:52.461+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*mQyPYu count=7616533","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.462410478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.462+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhooW31nH7WtVvyXGW5ddwYZUSQYrP8ogBiwoqwpCVEkGs count=7643379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.462445927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.462+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*CVEkGs count=7643380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.462062563Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:52.461+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*mQyPYu count=7616533","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466557056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.466+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466591348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.466+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CVEkGs inflight=1 count=7643382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466620469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.466+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CVEkGs inflight=0 count=7643383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466519886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*CVEkGs count=7643380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466519886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.466+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616534","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"j26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466556011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.466+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*mQyPYu inflight=1 count=7616535","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466570459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.466+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*mQyPYu inflight=0 count=7616536","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513219670Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.512+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoGU8NpybgCqa3wok6mbPSLqTQ8A4P75c1MYENehxjUGs - count=7643384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Em6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513255482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.513+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*hxjUGs count=7643385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513220679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*mQyPYu inflight=0 count=7616536","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466557056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.466+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466591348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.466+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*CVEkGs inflight=1 count=7643382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.466620469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.466+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*CVEkGs inflight=0 count=7643383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292466]},{"_index":"continuous-tests-pods-2023.11.10","_id":"kW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513220679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.512+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhn67puKVGZdARh6WPSpXTZVd5VppBM2PboukJP7e9G9Yu count=7616537","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"km6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513259330Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.512+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*e9G9Yu count=7616538","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"k26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.517597433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*e9G9Yu count=7616538","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513219670Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.512+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhoGU8NpybgCqa3wok6mbPSLqTQ8A4P75c1MYENehxjUGs + count=7643384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Em6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.513255482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.513+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*hxjUGs count=7643385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"k26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.517597433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.517+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616539","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292517]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.517641352Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.517+00:00 Advertising block topics=\"codex @@ -2450,17 +2450,17 @@ interactions: 2023-11-10 05:14:52.519+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*hxjUGs inflight=1 count=7643387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292519]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.519987680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.519+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*hxjUGs inflight=0 count=7643388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292519]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564416107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.564+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rheLhfb6R1wjhSqSg1GrcucRkXTG3w5SpfabB45PRH3KGs - count=7643389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"F26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564463836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.564+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*RH3KGs count=7643390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564302411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*hxjUGs inflight=0 count=7643388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292519]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564302411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.563+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhhdTqSgckgC8nnuKfZTDjFYyUhtxrF3BMkpMYGjhxbdYu count=7616542","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"l26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564341715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.564+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*hxbdYu count=7616543","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.568929207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*hxbdYu count=7616543","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564416107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.564+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rheLhfb6R1wjhSqSg1GrcucRkXTG3w5SpfabB45PRH3KGs + count=7643389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"F26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.564463836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.564+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*RH3KGs count=7643390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292564]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.568929207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.568+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292568]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.568977349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.568+00:00 Advertising block topics=\"codex @@ -2516,17 +2516,17 @@ interactions: 2023-11-10 05:14:52.671+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*cnZ9Yu inflight=1 count=7616555","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292671]},{"_index":"continuous-tests-pods-2023.11.10","_id":"pG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.671394300Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.671+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*cnZ9Yu inflight=0 count=7616556","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292671]},{"_index":"continuous-tests-pods-2023.11.10","_id":"JW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717202327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.716+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhb7tSwCgKz19GTf85rCJcgsCUKUPnFL44goDSNNoy2hGs - count=7643404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Jm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717310082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.716+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*oy2hGs count=7643405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"pW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717040656Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*cnZ9Yu inflight=0 count=7616556","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292671]},{"_index":"continuous-tests-pods-2023.11.10","_id":"pW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717040656Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.716+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rheuQFfHJFUicDNzrSb2tMjvnnaG9URKSB95HtpqM9zUYu count=7616557","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"pm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717096695Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.716+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*M9zUYu count=7616558","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"J26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.721288201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*M9zUYu count=7616558","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"JW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717202327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.716+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhb7tSwCgKz19GTf85rCJcgsCUKUPnFL44goDSNNoy2hGs + count=7643404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Jm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.717310082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.716+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*oy2hGs count=7643405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292717]},{"_index":"continuous-tests-pods-2023.11.10","_id":"J26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.721288201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.720+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292721]},{"_index":"continuous-tests-pods-2023.11.10","_id":"KG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.721326574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.721+00:00 Advertising block topics=\"codex @@ -2541,14 +2541,14 @@ interactions: discoveryengine\" tid=1 cid=zb2*M9zUYu inflight=0 count=7616561","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292722]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Km6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.767971884Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.767+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhZ6tbZ7S964hEuXgKK1dFtCUN8nj5vP5yo2RQWVRnFvGs - count=7643409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292767]},{"_index":"continuous-tests-pods-2023.11.10","_id":"K26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.768041698Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.767+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*RnFvGs count=7643410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292768]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.768067222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + count=7643409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292767]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.768067222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.767+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rheA9T5b2FWNCaog5MegZUrwB3oiuK37dXoUkzvjWiDDYu count=7616562","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292768]},{"_index":"continuous-tests-pods-2023.11.10","_id":"q26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.768142826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.767+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*WiDDYu count=7616563","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292768]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.772390213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*WiDDYu count=7616563","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292768]},{"_index":"continuous-tests-pods-2023.11.10","_id":"K26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.768041698Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.767+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*RnFvGs count=7643410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292768]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.772390213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.772+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292772]},{"_index":"continuous-tests-pods-2023.11.10","_id":"LW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.772440577Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.772+00:00 Advertising block topics=\"codex @@ -2585,14 +2585,14 @@ interactions: discoveryengine\" tid=1 cid=zb2*n8GxYu inflight=0 count=7616571","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292826]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.869908265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.869+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhigLYjZkPSKfkb9GU1fLLmBzC6DZ2L631HXVJ1cAct9Gs - count=7643419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292869]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.870056129Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.869+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*Act9Gs count=7643420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.870121544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + count=7643419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292869]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.870121544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.869+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhoP7BkuoLSU3bAfSh8vsX47fDBA1wS7d6aXxcynXYiCYu count=7616572","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.870172233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.869+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*XYiCYu count=7616573","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.874243789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*XYiCYu count=7616573","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"NW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.870056129Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.869+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*Act9Gs count=7643420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.874243789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.873+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292874]},{"_index":"continuous-tests-pods-2023.11.10","_id":"N26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.874278137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.874+00:00 Advertising block topics=\"codex @@ -2604,29 +2604,29 @@ interactions: 2023-11-10 05:14:52.875+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*XYiCYu inflight=1 count=7616575","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292875]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.875628498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.875+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*XYiCYu inflight=0 count=7616576","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292875]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921020085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.920+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhmZSdkyFD8vdJVhr5adATLoCXvX7xXyFAizBYvScxNBGs - count=7643424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Om6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921156051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.920+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*cxNBGs count=7643425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921236912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*XYiCYu inflight=0 count=7616576","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292875]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921236912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.920+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhkRjc8AQvvhmrzb39ieFty1tNn4WNvWyGzw4Kw9srxxYu count=7616577","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"um6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921278024Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.920+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*srxxYu count=7616578","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"O26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.926959829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*srxxYu count=7616578","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921020085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.920+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhmZSdkyFD8vdJVhr5adATLoCXvX7xXyFAizBYvScxNBGs + count=7643424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Om6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.921156051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.920+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*cxNBGs count=7643425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292921]},{"_index":"continuous-tests-pods-2023.11.10","_id":"O26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.926959829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.926+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292926]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927005812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.926+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*cxNBGs inflight=1 count=7643427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927027368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:52.926+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*cxNBGs inflight=0 count=7643428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"u26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927186958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 nodes=1 count=7643426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292926]},{"_index":"continuous-tests-pods-2023.11.10","_id":"u26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927186958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.926+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616579","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927227511Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.926+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*srxxYu inflight=1 count=7616580","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927453022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.927+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*srxxYu inflight=0 count=7616581","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.971734657Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*srxxYu inflight=0 count=7616581","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927005812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.926+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cxNBGs inflight=1 count=7643427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"PW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.927027368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:52.926+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cxNBGs inflight=0 count=7643428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292927]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Pm6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.971734657Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:52.971+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Gs/zb2rhhnUoFi7bsXNLvhoMez2A1tx6rTcT1Frx123LSjLZo6Gs count=7643429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593292971]},{"_index":"continuous-tests-pods-2023.11.10","_id":"P26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.971817044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -2648,23 +2648,23 @@ interactions: 2023-11-10 05:14:52.977+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*B6vMYu inflight=1 count=7616585","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292978]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:52.978100252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:52.977+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*B6vMYu inflight=0 count=7616586","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292978]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022612574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:53.022+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/AV/zb2rhZdFAioRwRaiBNoSDgMWHQScgRjm4PtPm5E6iWHTRh4AV - count=7643434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022644311Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:53.022+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*TRh4AV count=7643435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022928633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*B6vMYu inflight=0 count=7616586","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593292978]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w26nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022928633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:53.022+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Yu/zb2rhfCB4CuTCXdPQ5qXsK5iNH7yjyCBxPTRCXdwkRVm862Yu count=7616587","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xG6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022998222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:53.022+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*m862Yu count=7616588","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.026678336Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:53.026+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7643436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293026]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zm6nt4sBOvK4VyD1nMOx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.026711794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*m862Yu count=7616588","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Q26nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022612574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:53.022+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/AV/zb2rhZdFAioRwRaiBNoSDgMWHQScgRjm4PtPm5E6iWHTRh4AV + count=7643434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RG6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.022644311Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:53.022+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*TRh4AV count=7643435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293022]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zm6nt4sBOvK4VyD1nMOx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.026711794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:53.026+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*TRh4AV inflight=1 count=7643437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293026]},{"_index":"continuous-tests-pods-2023.11.10","_id":"z26nt4sBOvK4VyD1nMOx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.026761768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:53.026+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*TRh4AV inflight=0 count=7643438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293026]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.027634659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*TRh4AV inflight=0 count=7643438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293026]},{"_index":"continuous-tests-pods-2023.11.10","_id":"RW6nt4sBOvK4VyD1jSMC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.026678336Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:53.026+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7643436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593293026]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xW6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.027634659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:53.027+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7616589","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593293027]},{"_index":"continuous-tests-pods-2023.11.10","_id":"xm6nt4sBOvK4VyD1lm3o","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:53.027721810Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:53.027+00:00 Advertising block topics=\"codex @@ -4397,13 +4397,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/ce/zb2rhkaD6kAeowLeTkv4txRqxgdBuinERLuft7cX4u9sVgace count=7643831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297044]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1nMWx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.044800253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:57.044+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*sVgace count=7643832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297044]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1rOyo","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.049753031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:14:57.049+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*sVgace inflight=0 count=7643835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297049]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1nMWx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.049709374Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*sVgace count=7643832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297044]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm6nt4sBOvK4VyD1nMWx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.049709374Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:57.049+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7643833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297049]},{"_index":"continuous-tests-pods-2023.11.10","_id":"W26nt4sBOvK4VyD1nMWx","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.049747243Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:57.049+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*sVgace inflight=1 count=7643834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297049]},{"_index":"continuous-tests-pods-2023.11.10","_id":"c26nt4sBOvK4VyD1puNz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.056245632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*sVgace inflight=1 count=7643834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297049]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D26nt4sBOvK4VyD1rOyo","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.049753031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:14:57.049+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*sVgace inflight=0 count=7643835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593297049]},{"_index":"continuous-tests-pods-2023.11.10","_id":"c26nt4sBOvK4VyD1puNz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.056245632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:57.055+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/cr/zb2rhYcRuaiv8kqVn8F2G3AmpZaZrqdWDgR59WEekLSXaXocr count=7616983","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593297056]},{"_index":"continuous-tests-pods-2023.11.10","_id":"dG6nt4sBOvK4VyD1puNz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:57.056285740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -5490,13 +5490,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/uX/zb2rhbkN9GdrxitbgZ3ALc6YmF34zbdgy5gGmMEQyzpmTr4uX count=7617229","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299550]},{"_index":"continuous-tests-pods-2023.11.10","_id":"am6nt4sBOvK4VyD1puRz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.550519183Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:59.550+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*mTr4uX count=7617230","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299550]},{"_index":"continuous-tests-pods-2023.11.10","_id":"a26nt4sBOvK4VyD1puRz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.555691349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*mTr4uX count=7617230","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299550]},{"_index":"continuous-tests-pods-2023.11.10","_id":"i26nt4sBOvK4VyD1tvZC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.555735920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:14:59.555+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*mTr4uX inflight=0 count=7617233","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299555]},{"_index":"continuous-tests-pods-2023.11.10","_id":"a26nt4sBOvK4VyD1puRz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.555691349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:59.555+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7617231","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299555]},{"_index":"continuous-tests-pods-2023.11.10","_id":"bG6nt4sBOvK4VyD1puRz","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.555727227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:14:59.555+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*mTr4uX inflight=1 count=7617232","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299555]},{"_index":"continuous-tests-pods-2023.11.10","_id":"i26nt4sBOvK4VyD1tvZC","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.555735920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:14:59.555+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*mTr4uX inflight=0 count=7617233","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299555]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Bm6nt4sBOvK4VyD1rO2p","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.594499748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*mTr4uX inflight=1 count=7617232","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593299555]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Bm6nt4sBOvK4VyD1rO2p","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.594499748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:14:59.593+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/n3/zb2rhfx6LuuyXVdfjARWkxjp3Y5wXtj2egRmyw5y9gT2iM5n3 count=7644082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593299594]},{"_index":"continuous-tests-pods-2023.11.10","_id":"B26nt4sBOvK4VyD1rO2p","_score":null,"_source":{"@timestamp":"2023-11-10T05:14:59.594539946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -6389,14 +6389,14 @@ interactions: 2023-11-10 05:15:01.643+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7644284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301643]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0W6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.643808151Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:01.643+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*9Fhsn3 inflight=1 count=7644285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301643]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0m6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.644075329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:01.643+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*9Fhsn3 inflight=0 count=7644286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301644]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Vm6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.644376515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*9Fhsn3 inflight=1 count=7644285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301643]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Vm6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.644376515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.643+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhcgJLgBydQwYzZjwYB2nxYzd6a4PkxnVkwNcb4utKXkQb count=7617436","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301644]},{"_index":"continuous-tests-pods-2023.11.10","_id":"V26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.644464283Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.644+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*tKXkQb count=7617437","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301644]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.651780542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*tKXkQb count=7617437","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301644]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0m6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.644075329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:01.643+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*9Fhsn3 inflight=0 count=7644286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301644]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.651780542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.651+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7617438","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301651]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.651828228Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.651+00:00 Advertising block topics=\"codex @@ -6407,16 +6407,16 @@ interactions: repostore\" tid=1 cid=/repo/blocks/n3/zb2rhjh7SjseXcfG5Wi1z1c4mDiKbAso5WjhH6gTy78VPCxn3 count=7644287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301688]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1G6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.688701679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:01.688+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*VPCxn3 count=7644288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301688]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1W6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695781086Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:01.695+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7644289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1m6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695820076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:01.695+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*VPCxn3 inflight=1 count=7644290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"W26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695495728Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*VPCxn3 count=7644288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301688]},{"_index":"continuous-tests-pods-2023.11.10","_id":"W26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695495728Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.695+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhdUqJxkMciiGNCJHFXhm6VLs1WtiyXueZNrZM3ygNd2Qb count=7617441","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695613068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.695+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*gNd2Qb count=7617442","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"126nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.696028235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*gNd2Qb count=7617442","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1W6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695781086Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:01.695+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7644289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1m6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.695820076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:01.695+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*VPCxn3 inflight=1 count=7644290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301695]},{"_index":"continuous-tests-pods-2023.11.10","_id":"126nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.696028235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:01.695+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*VPCxn3 inflight=0 count=7644291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301696]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.704570420Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.704+00:00 Provided to nodes topics=\"codex @@ -6433,14 +6433,14 @@ interactions: 2023-11-10 05:15:01.745+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7644294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301745]},{"_index":"continuous-tests-pods-2023.11.10","_id":"226nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.745869253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:01.745+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*8uvkn3 inflight=1 count=7644295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301745]},{"_index":"continuous-tests-pods-2023.11.10","_id":"3G6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.746161817Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:01.745+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*8uvkn3 inflight=0 count=7644296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301746]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.746775137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*8uvkn3 inflight=1 count=7644295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301745]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.746775137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.746+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhdRJtHKeEPDLQQvHnDxmXnHgny9EbT57iZo3pGaxe7aQb count=7617446","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301746]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YW6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.746832196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.746+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*xe7aQb count=7617447","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301746]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Ym6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.751781463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*xe7aQb count=7617447","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301746]},{"_index":"continuous-tests-pods-2023.11.10","_id":"3G6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.746161817Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:01.745+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*8uvkn3 inflight=0 count=7644296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301746]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Ym6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.751781463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.751+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7617448","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593301751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Y26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.751847142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:01.751+00:00 Advertising block topics=\"codex @@ -6539,18 +6539,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/n3/zb2rheabYQ6JMTrQ4ccyyuG2RizesrVacKbDMnuRfvjzLi5n3 count=7644317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301994]},{"_index":"continuous-tests-pods-2023.11.10","_id":"8m6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:01.994161260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:01.993+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*zLi5n3 count=7644318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301994]},{"_index":"continuous-tests-pods-2023.11.10","_id":"826nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002073915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*zLi5n3 count=7644318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593301994]},{"_index":"continuous-tests-pods-2023.11.10","_id":"eW6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002385523Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:02.002+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhZAuJAyPMCH1CSNy5u78Y2q3XEQC1ZJMungnABAw3h5Qb + count=7617471","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"em6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002518846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:02.002+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*w3h5Qb count=7617472","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"826nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002073915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:02.000+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7644319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9G6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002118592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:02.000+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*zLi5n3 inflight=1 count=7644320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9W6nt4sBOvK4VyD1vP-f","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002151880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:02.000+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*zLi5n3 inflight=0 count=7644321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"eW6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002385523Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:02.002+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhZAuJAyPMCH1CSNy5u78Y2q3XEQC1ZJMungnABAw3h5Qb - count=7617471","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"em6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.002518846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:02.002+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*w3h5Qb count=7617472","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"e26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.008620223Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*zLi5n3 inflight=0 count=7644321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593302002]},{"_index":"continuous-tests-pods-2023.11.10","_id":"e26nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.008620223Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:02.008+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7617473","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593302008]},{"_index":"continuous-tests-pods-2023.11.10","_id":"fG6nt4sBOvK4VyD1tvdC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:02.008704815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:02.008+00:00 Advertising block topics=\"codex @@ -7220,12 +7220,12 @@ interactions: 2023-11-10 05:15:03.567+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*Ysyux9 inflight=1 count=7644471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593303568]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jG-nt4sBOvK4VyD1vACf","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.568256471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:03.567+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*Ysyux9 inflight=0 count=7644472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593303568]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AG-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.582288907Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:03.581+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*tiAgQb count=7617628","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593303582]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW6nt4sBOvK4VyD1tvhC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.582180835Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*Ysyux9 inflight=0 count=7644472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593303568]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW6nt4sBOvK4VyD1tvhC","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.582180835Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:03.581+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Qb/zb2rhcDLftgZy1VC8FruooxDL6tNemPhBAJKphhQrvitiAgQb - count=7617627","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593303582]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AW-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.587758320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + count=7617627","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593303582]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AG-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.582288907Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:03.581+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*tiAgQb count=7617628","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593303582]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AW-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.587758320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:03.587+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7617629","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593303587]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Am-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:03.587815928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:03.587+00:00 Advertising block topics=\"codex @@ -7918,13 +7918,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/x9/zb2rheKEbdKBDfVmhZUWQB7t6MQRhuSPiS2QyYRcXTR8RDvx9 count=7644628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305197]},{"_index":"continuous-tests-pods-2023.11.10","_id":"KW-nt4sBOvK4VyD1vAGf","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.197685453Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:05.197+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*8RDvx9 count=7644629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305197]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Km-nt4sBOvK4VyD1vAGf","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.208831960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:05.208+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7644630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305208]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CG-nt4sBOvK4VyD1zBFu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.208877532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*8RDvx9 count=7644629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305197]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CG-nt4sBOvK4VyD1zBFu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.208877532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:05.208+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*8RDvx9 inflight=1 count=7644631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305208]},{"_index":"continuous-tests-pods-2023.11.10","_id":"CW-nt4sBOvK4VyD1zBFu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.208937194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:05.208+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*8RDvx9 inflight=0 count=7644632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305208]},{"_index":"continuous-tests-pods-2023.11.10","_id":"n2-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.221605334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*8RDvx9 inflight=0 count=7644632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305208]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Km-nt4sBOvK4VyD1vAGf","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.208831960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:05.208+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7644630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593305208]},{"_index":"continuous-tests-pods-2023.11.10","_id":"n2-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.221605334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:05.221+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/6w/zb2rhjLEDKjzjQybqNGcTPgsuA4JWBYpBN3b3xsJaVx1BP26w count=7617787","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593305221]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oG-nt4sBOvK4VyD1xQj7","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:05.221655351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -9319,13 +9319,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/zq/zb2rhimmkvSQP7CQ9B9mk3Wk68US6m3dzzSkRbMVdKaY5qezq count=7618103","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308434]},{"_index":"continuous-tests-pods-2023.11.10","_id":"im-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.434116620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.433+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*Y5qezq count=7618104","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308434]},{"_index":"continuous-tests-pods-2023.11.10","_id":"i2-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.438693318Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*Y5qezq count=7618104","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308434]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm-nt4sBOvK4VyD1zBJu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.438140685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:08.437+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=7644949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308438]},{"_index":"continuous-tests-pods-2023.11.10","_id":"i2-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.438693318Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.438+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7618105","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308438]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jG-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.438736953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.438+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*Y5qezq inflight=1 count=7618106","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308438]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm-nt4sBOvK4VyD1zBJu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.438140685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:08.437+00:00 About to sleep discovery loop topics=\"codex - discoveryengine\" tid=1 sleep=3s wanted=0 count=7644949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308438]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jW-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.439141680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*Y5qezq inflight=1 count=7618106","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308438]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jW-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.439141680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.438+00:00 Advertised block topics=\"codex discoveryengine\" tid=1 cid=zb2*Y5qezq inflight=0 count=7618107","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308439]},{"_index":"continuous-tests-pods-2023.11.10","_id":"R2-nt4sBOvK4VyD1zBJu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.462622277Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:08.462+00:00 Retrieved record from repo topics=\"codex @@ -9508,18 +9508,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/GZ/zb2rhforPZzXWQbz5Ssdw4U8iSp75XCUg7758SFANT9wXueGZ count=7644990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cG-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.870706061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:08.869+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*wXueGZ count=7644991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tm-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891575346Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:08.891+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/zq/zb2rhgmLYUd4zKCYfndbZch9WNEi2xcnbnXP1jCgFXodJTwzq - count=7618148","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"t2-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891798046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:08.891+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*dJTwzq count=7618149","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cW-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891466229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*wXueGZ count=7644991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308870]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cW-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891466229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:08.891+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7644992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cm-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891564596Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:08.891+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*wXueGZ inflight=1 count=7644993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"c2-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891578921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:08.891+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*wXueGZ inflight=0 count=7644994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uG-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.896188793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*wXueGZ inflight=0 count=7644994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"tm-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891575346Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:08.891+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/zq/zb2rhgmLYUd4zKCYfndbZch9WNEi2xcnbnXP1jCgFXodJTwzq + count=7618148","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"t2-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.891798046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:08.891+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*dJTwzq count=7618149","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308891]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uG-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.896188793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.895+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7618150","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593308896]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:08.896213480Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:08.896+00:00 Advertising block topics=\"codex @@ -9662,13 +9662,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/GS/zb2rha6baDrCQmtidFYsUmvRfVz3GNeYkiFaBmVk3Dk3tpkGS count=7645025","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309227]},{"_index":"continuous-tests-pods-2023.11.10","_id":"k2-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.228122415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:09.227+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*3tpkGS count=7645026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309228]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW-nt4sBOvK4VyD13CRM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.237129543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:09.236+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*3tpkGS inflight=0 count=7645029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lG-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.237082195Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*3tpkGS count=7645026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309228]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lG-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.237082195Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:09.236+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7645027","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"lW-nt4sBOvK4VyD1zBJv","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.237116644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:09.236+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*3tpkGS inflight=1 count=7645028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2W-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.250947274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*3tpkGS inflight=1 count=7645028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"uW-nt4sBOvK4VyD13CRM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.237129543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:09.236+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3tpkGS inflight=0 count=7645029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593309237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2W-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.250947274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:09.250+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/zq/zb2rheNRauD5sqVHyBxcKit559HAoboeVujbt9Xhd6whjbKzq count=7618183","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593309250]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2m-nt4sBOvK4VyD11R6W","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:09.250997986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -10691,13 +10691,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/j8/zb2rhj8TCXGmx1prNk16o2kCFQX9RXoeF2TSBGbDPT26nYrj8 count=7618415","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311599]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wm-nt4sBOvK4VyD11R-X","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.599346585Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:11.598+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*6nYrj8 count=7618416","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311599]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w2-nt4sBOvK4VyD11R-X","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.604676390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:11.604+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7618417","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311604]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XG-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.604717431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*6nYrj8 count=7618416","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311599]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XG-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.604717431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:11.604+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*6nYrj8 inflight=1 count=7618418","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311604]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.604738418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:11.604+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*6nYrj8 inflight=0 count=7618419","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311604]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oW-nt4sBOvK4VyD13CVM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.627691360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*6nYrj8 inflight=0 count=7618419","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311604]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w2-nt4sBOvK4VyD11R-X","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.604676390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:11.604+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7618417","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593311604]},{"_index":"continuous-tests-pods-2023.11.10","_id":"oW-nt4sBOvK4VyD13CVM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.627691360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:11.627+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/GS/zb2rhbdtuHWyy1SGV6aNLh9my1EFkAWWZfN9YjqLyL1ud3ZGS count=7645261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593311627]},{"_index":"continuous-tests-pods-2023.11.10","_id":"om-nt4sBOvK4VyD13CVM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:11.627735451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -11050,7 +11050,7 @@ interactions: code: 200 message: OK - request: - body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 @@ -11066,7 +11066,7 @@ interactions: uri: http://localhost:9200/_search/scroll response: body: - string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":104,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"qW-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:12.411791302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":289,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"qW-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:12.411791302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:12.411+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/j8/zb2rhe4ZiVq7AYNpuWpWp46DsjAMGx4v9UkxFMX43gRXvAYj8 count=7618495","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593312411]},{"_index":"continuous-tests-pods-2023.11.10","_id":"qm-nt4sBOvK4VyD15TNI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:12.411845006Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -11453,12 +11453,12 @@ interactions: 2023-11-10 05:15:13.282+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*J8FnmY inflight=1 count=7618584","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593313282]},{"_index":"continuous-tests-pods-2023.11.10","_id":"A2-nt4sBOvK4VyD15TRI","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.282855110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:13.282+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*J8FnmY inflight=0 count=7618585","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593313282]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm-nt4sBOvK4VyD13CZM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.311348477Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*J8FnmY inflight=0 count=7618585","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593313282]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cG-nt4sBOvK4VyD17D8H","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.311389475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:13.311+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*3jFGz4 count=7645427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593313311]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Rm-nt4sBOvK4VyD13CZM","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.311348477Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:13.311+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/z4/zb2rhWstjPFRxGdb3QFRAZAEQgs8KUKSk4EjYyP5xU83jFGz4 - count=7645426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593313311]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cG-nt4sBOvK4VyD17D8H","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.311389475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:13.311+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*3jFGz4 count=7645427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593313311]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cW-nt4sBOvK4VyD17D8H","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.316128142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + count=7645426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593313311]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cW-nt4sBOvK4VyD17D8H","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.316128142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:13.315+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7645428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593313316]},{"_index":"continuous-tests-pods-2023.11.10","_id":"cm-nt4sBOvK4VyD17D8H","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:13.316161231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:13.315+00:00 Advertising block topics=\"codex @@ -15962,13 +15962,13 @@ interactions: 2023-11-10 05:15:23.834+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*Xc3mSc inflight=1 count=7646458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593323834]},{"_index":"continuous-tests-pods-2023.11.10","_id":"w2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.834476112Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:23.834+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*Xc3mSc inflight=0 count=7646459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593323834]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.835289833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*Xc3mSc inflight=0 count=7646459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593323834]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2G-ot4sBOvK4VyD1BZhZ","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.835240592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:23.834+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7619602","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323835]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vW-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.835289833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:23.835+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*Z8BVLM inflight=1 count=7619603","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323835]},{"_index":"continuous-tests-pods-2023.11.10","_id":"vm-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.835304363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:23.835+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*Z8BVLM inflight=0 count=7619604","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323835]},{"_index":"continuous-tests-pods-2023.11.10","_id":"2G-ot4sBOvK4VyD1BZhZ","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.835240592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:23.834+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7619602","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323835]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v2-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.879955785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*Z8BVLM inflight=0 count=7619604","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323835]},{"_index":"continuous-tests-pods-2023.11.10","_id":"v2-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.879955785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:23.879+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/LM/zb2rhXk2tTEqqK1P4zENMh1WAfkuSVC8JUXhxyrx2fEivaSLM count=7619605","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323879]},{"_index":"continuous-tests-pods-2023.11.10","_id":"wG-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.879995392Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -16026,12 +16026,7 @@ interactions: 2023-11-10 05:15:23.985+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*CHm1LM inflight=1 count=7619618","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323985]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zW-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:23.986006394Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:23.985+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*CHm1LM inflight=0 count=7619619","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323986]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zm-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032054923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.031+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhm5DqXjUhLjeYYFNBrdt4PJTEj3AG7Ltvy6SveuLghsLM - count=7619620","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"z2-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032132997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.031+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*LghsLM count=7619621","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"z2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032609492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*CHm1LM inflight=0 count=7619619","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593323986]},{"_index":"continuous-tests-pods-2023.11.10","_id":"z2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032609492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:23.983+00:00 Providing block topics=\"codex discovery\" tid=1 cid=zb2*DxvtSc count=7646471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0G-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032640689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.012+00:00 Provided to nodes topics=\"codex @@ -16039,7 +16034,12 @@ interactions: 2023-11-10 05:15:24.012+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*DxvtSc inflight=1 count=7646473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"0m-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032652168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.012+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*DxvtSc inflight=0 count=7646474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"02-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.033701361Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*DxvtSc inflight=0 count=7646474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"zm-ot4sBOvK4VyD1Fa0h","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032054923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.031+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhm5DqXjUhLjeYYFNBrdt4PJTEj3AG7Ltvy6SveuLghsLM + count=7619620","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"z2-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.032132997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.031+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*LghsLM count=7619621","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324032]},{"_index":"continuous-tests-pods-2023.11.10","_id":"02-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.033701361Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.033+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Sc/zb2rhhkNmKDbNNHt8CEehTLneJcRBbK1zvpWK9rGDWXyQ7YSc count=7646475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324033]},{"_index":"continuous-tests-pods-2023.11.10","_id":"1G-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.034117034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -16127,18 +16127,18 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Sc/zb2rhZ1VUQ2Dp8M9DS8mMrAWZiSFQ2K5EXqu19ggV3m1M2ySc count=7646495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6G-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.237771803Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.237+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*1M2ySc count=7646496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242687349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.239+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhi91yVRMEqvKDeeuQV4WxYj5rM5zz1cG9T1WYrxhGiTLM - count=7619640","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"42-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242731930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.239+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*hGiTLM count=7619641","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6W-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242638964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*1M2ySc count=7646496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324237]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6W-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242638964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.242+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"6m-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242667037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.242+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*1M2ySc inflight=1 count=7646498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"62-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242671517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.242+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*1M2ySc inflight=0 count=7646499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"5G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.248632011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*1M2ySc inflight=0 count=7646499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"4m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242687349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.239+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhi91yVRMEqvKDeeuQV4WxYj5rM5zz1cG9T1WYrxhGiTLM + count=7619640","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"42-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.242731930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.239+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*hGiTLM count=7619641","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324242]},{"_index":"continuous-tests-pods-2023.11.10","_id":"5G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.248632011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.248+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7619642","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324248]},{"_index":"continuous-tests-pods-2023.11.10","_id":"5W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.248695364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.248+00:00 Advertising block topics=\"codex @@ -16210,41 +16210,41 @@ interactions: 2023-11-10 05:15:24.401+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*vdKhLM inflight=1 count=7619658","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324401]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.401646467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.401+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*vdKhLM inflight=0 count=7619659","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324401]},{"_index":"continuous-tests-pods-2023.11.10","_id":"92-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445429794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.445+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*jjXzLM count=7619661","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445290974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.444+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhmfeHaXmKSGAJa1gkwg5vXrj5icfM56XLouUF3GjjXzLM - count=7619660","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445915536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*vdKhLM inflight=0 count=7619659","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324401]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445915536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.445+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Sc/zb2rhkefE6dpSXWGbUxWiWXaYFnyGxQ9UkzfgYV7MUaD718Sc count=7646515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_G-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445959352Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.445+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*D718Sc count=7646516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451607885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.451+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7619662","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451662284Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.451+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*jjXzLM inflight=1 count=7619663","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451728281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.451+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*jjXzLM inflight=0 count=7619664","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451214683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*D718Sc count=7646516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"92-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445429794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.445+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*jjXzLM count=7619661","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"9m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.445290974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.444+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhmfeHaXmKSGAJa1gkwg5vXrj5icfM56XLouUF3GjjXzLM + count=7619660","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324445]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451214683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.450+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_m-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451250463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.451+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*D718Sc inflight=1 count=7646518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_2-ot4sBOvK4VyD1C6P9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451255628Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.451+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*D718Sc inflight=0 count=7646519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-2-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.474215207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*D718Sc inflight=0 count=7646519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451607885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.451+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7619662","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451662284Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.451+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*jjXzLM inflight=1 count=7619663","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.451728281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.451+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*jjXzLM inflight=0 count=7619664","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324451]},{"_index":"continuous-tests-pods-2023.11.10","_id":"-2-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.474215207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.473+00:00 About to sleep discovery loop topics=\"codex - discoveryengine\" tid=1 sleep=3s wanted=0 count=7619665","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324474]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496570914Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.496+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhjYLUGzmAioWN9zSadDuh4gCDv1wMNoEFVHTWQcoTsYLM - count=7619666","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496620588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.496+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*oTsYLM count=7619667","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496806672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 sleep=3s wanted=0 count=7619665","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324474]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496806672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.496+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/Sc/zb2rhd68jaS7PGyGqx8soevbrB9Y8w1Qj889HYy9NPbXoERSc count=7646520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"AW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496842761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.496+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*XoERSc count=7646521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.501919182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discovery\" tid=1 cid=zb2*XoERSc count=7646521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_G-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496570914Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.496+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhjYLUGzmAioWN9zSadDuh4gCDv1wMNoEFVHTWQcoTsYLM + count=7619666","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_W-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.496620588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.496+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*oTsYLM count=7619667","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324496]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_m-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.501919182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.501+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7619668","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324501]},{"_index":"continuous-tests-pods-2023.11.10","_id":"_2-ot4sBOvK4VyD1Fa0i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.501950974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.501+00:00 Advertising block topics=\"codex @@ -16310,29 +16310,29 @@ interactions: repostore\" tid=1 cid=/repo/blocks/Sc/zb2rhob4EA4vbsrgCWR2uGq9HJYoTu7nEStRU6AVYcWcUAGSc count=7646535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324650]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.650079704Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.649+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*cUAGSc count=7646536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324650]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655912955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.655+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7619683","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655988921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.655+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*xcTeLM inflight=1 count=7619684","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655042546Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*cUAGSc count=7646536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324650]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655042546Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.654+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Em-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655124890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.654+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*cUAGSc inflight=1 count=7646538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"E2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655131473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.655+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*cUAGSc inflight=0 count=7646539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D2-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.656014916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*cUAGSc inflight=0 count=7646539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"DW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655912955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.655+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7619683","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Dm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.655988921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.655+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*xcTeLM inflight=1 count=7619684","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324655]},{"_index":"continuous-tests-pods-2023.11.10","_id":"D2-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.656014916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.655+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*xcTeLM inflight=0 count=7619685","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324656]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700540483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.700+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhjewrj6mUyNm2pARwzwra6sCGX5TJkqa5yhG5jsoLk5LM - count=7619686","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700588117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.700+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*oLk5LM count=7619687","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700676363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*xcTeLM inflight=0 count=7619685","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324656]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700676363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.700+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/us/zb2rhcdRXQ8aKmgwD5LHvQofKdDS32PUVvQ5atrAg9RnJtXus count=7646540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700711971Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.700+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*nJtXus count=7646541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.705889575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*nJtXus count=7646541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700540483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.700+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhjewrj6mUyNm2pARwzwra6sCGX5TJkqa5yhG5jsoLk5LM + count=7619686","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"EW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.700588117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.700+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*oLk5LM count=7619687","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324700]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.705889575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.705+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324705]},{"_index":"continuous-tests-pods-2023.11.10","_id":"F2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.705921825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.705+00:00 Advertising block topics=\"codex @@ -16344,17 +16344,17 @@ interactions: 2023-11-10 05:15:24.706+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*oLk5LM inflight=1 count=7619689","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324706]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.706852163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:24.706+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*oLk5LM inflight=0 count=7619690","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324706]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751391498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.751+00:00 Retrieved record from repo topics=\"codex - repostore\" tid=1 cid=/repo/blocks/LM/zb2rhZi35t7SRkdEMLVwfUt58tLKHT2RKNmqQMHutgrY5ZoLM - count=7619691","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751513667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:24.751+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*Y5ZoLM count=7619692","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751498679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*oLk5LM inflight=0 count=7619690","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324706]},{"_index":"continuous-tests-pods-2023.11.10","_id":"GW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751498679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.751+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/us/zb2rhjBkrWjKcp1vyxCvgdz7Z5YubT6NsqzRdmTsgFFLaQtus count=7646545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Gm-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751604550Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.751+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*LaQtus count=7646546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"G2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.755500634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*LaQtus count=7646546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"FW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751391498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.751+00:00 Retrieved record from repo topics=\"codex + repostore\" tid=1 cid=/repo/blocks/LM/zb2rhZi35t7SRkdEMLVwfUt58tLKHT2RKNmqQMHutgrY5ZoLM + count=7619691","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Fm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.751513667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:24.751+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*Y5ZoLM count=7619692","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593324751]},{"_index":"continuous-tests-pods-2023.11.10","_id":"G2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.755500634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.755+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593324755]},{"_index":"continuous-tests-pods-2023.11.10","_id":"HG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:24.755536354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:24.755+00:00 Advertising block topics=\"codex @@ -16640,19 +16640,19 @@ interactions: repostore\" tid=1 cid=/repo/blocks/do/zb2rhkA8KPQae6tKyJSxrAkNc56bdZU1wiwTicNsgUU5ngFdo count=7619756","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325414]},{"_index":"continuous-tests-pods-2023.11.10","_id":"V2-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.414598169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:25.414+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*5ngFdo count=7619757","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325414]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419568102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:25.419+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*5ngFdo inflight=0 count=7619760","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419449682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:25.419+00:00 Provided to nodes topics=\"codex - discovery\" tid=1 nodes=1 count=7619758","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419543426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC - 2023-11-10 05:15:25.419+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*5ngFdo inflight=1 count=7619759","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419845314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*5ngFdo count=7619757","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325414]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419845314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:25.419+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7646612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"XW-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419880406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:25.419+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*iHNDus inflight=1 count=7646613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Xm-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419885994Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:25.419+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*iHNDus inflight=0 count=7646614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.462529065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*iHNDus inflight=0 count=7646614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Wm-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419568102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:25.419+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*5ngFdo inflight=0 count=7619760","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419449682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:25.419+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=7619758","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"WW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.419543426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + 2023-11-10 05:15:25.419+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*5ngFdo inflight=1 count=7619759","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325419]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X2-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.462529065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:25.461+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/us/zb2rhWnfuYq7LkpswEEa34kYEw1rym34795xpQNoQ6MRfBXus count=7646615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325462]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.462728334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC @@ -16674,12 +16674,12 @@ interactions: 2023-11-10 05:15:25.470+00:00 Advertising block topics=\"codex discoveryengine\" tid=1 cid=zb2*ZTwJdo inflight=1 count=7619764","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325471]},{"_index":"continuous-tests-pods-2023.11.10","_id":"X2-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.471258412Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:25.471+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*ZTwJdo inflight=0 count=7619765","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325471]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yG-ot4sBOvK4VyD1G7XD","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.513603642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:25.513+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*8dVEus count=7646621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"ZG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.513518233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*ZTwJdo inflight=0 count=7619765","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325471]},{"_index":"continuous-tests-pods-2023.11.10","_id":"ZG-ot4sBOvK4VyD1C6T9","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.513518233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:25.513+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/us/zb2rhcjTywRM96Uc3w8vK6swsfrrefdhNtaRSjjU8pV8dVEus - count=7646620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.516902386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + count=7646620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"yG-ot4sBOvK4VyD1G7XD","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.513603642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:25.513+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*8dVEus count=7646621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593325513]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YG-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.516902386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:25.516+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/do/zb2rhZgZgEULKzFRyKKrKqvEic51sgDYDjMRPAawisv97fDdo count=7619766","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593325516]},{"_index":"continuous-tests-pods-2023.11.10","_id":"YW-ot4sBOvK4VyD1Fa4i","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:25.517096795Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -22075,7 +22075,7 @@ interactions: code: 200 message: OK - request: - body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 @@ -22091,7 +22091,7 @@ interactions: uri: http://localhost:9200/_search/scroll response: body: - string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":47,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"NW-ot4sBOvK4VyD1ROYu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:38.381475114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":44,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.10","_id":"NW-ot4sBOvK4VyD1ROYu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:38.381475114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:38.381+00:00 Providing block topics=\"codex discovery\" tid=1 cid=zb2*Xdgcv2 count=7621032","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593338381]},{"_index":"continuous-tests-pods-2023.11.10","_id":"Nm-ot4sBOvK4VyD1ROYu","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:38.386756407Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:38.386+00:00 Provided to nodes topics=\"codex @@ -25627,13 +25627,13 @@ interactions: repostore\" tid=1 cid=/repo/blocks/JH/zb2rhXQvTeNLAv9eb5fhe2uMvrV2qGJ2zFvDCns7tczwEKpJH count=7648607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346542]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jHCot4sBOvK4VyD1XhcU","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.542520363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:46.542+00:00 Providing block topics=\"codex - discovery\" tid=1 cid=zb2*wEKpJH count=7648608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346542]},{"_index":"continuous-tests-pods-2023.11.10","_id":"7HCot4sBOvK4VyD1bloD","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.546442181Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC - 2023-11-10 05:15:46.546+00:00 Advertised block topics=\"codex - discoveryengine\" tid=1 cid=zb2*wEKpJH inflight=0 count=7648611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346546]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jXCot4sBOvK4VyD1XhcU","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.546392315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + discovery\" tid=1 cid=zb2*wEKpJH count=7648608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346542]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jXCot4sBOvK4VyD1XhcU","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.546392315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:46.545+00:00 Provided to nodes topics=\"codex discovery\" tid=1 nodes=1 count=7648609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346546]},{"_index":"continuous-tests-pods-2023.11.10","_id":"jnCot4sBOvK4VyD1XhcU","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.546423570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC 2023-11-10 05:15:46.546+00:00 Advertising block topics=\"codex - discoveryengine\" tid=1 cid=zb2*wEKpJH inflight=1 count=7648610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346546]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OXCot4sBOvK4VyD1Yy1q","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.548924308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC + discoveryengine\" tid=1 cid=zb2*wEKpJH inflight=1 count=7648610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346546]},{"_index":"continuous-tests-pods-2023.11.10","_id":"7HCot4sBOvK4VyD1bloD","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.546442181Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-10 05:15:46.546+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wEKpJH inflight=0 count=7648611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699593346546]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OXCot4sBOvK4VyD1Yy1q","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.548924308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC 2023-11-10 05:15:46.548+00:00 Retrieved record from repo topics=\"codex repostore\" tid=1 cid=/repo/blocks/PZ/zb2rhYzm7B22jpW5AEJ1PVdwJGQfdU3iCNmwqeYYSEgqKwVPZ count=7621835","pod_ip":"10.244.2.250","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"58b69484bc","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"bootstrap-2-58b69484bc-88msf","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcblu"},"sort":[1699593346548]},{"_index":"continuous-tests-pods-2023.11.10","_id":"OnCot4sBOvK4VyD1Yy1q","_score":null,"_source":{"@timestamp":"2023-11-10T05:15:46.548988279Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"bootstrap-2","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_bootstrap-2-58b69484bc-88msf_5fc7ae5d-b7c8-4628-9bf2-a70e8b6934ae/bootstrap-2/0.log","message":"TRC @@ -25876,7 +25876,7 @@ interactions: code: 200 message: OK - request: - body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","scroll":"2m"}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 @@ -25892,7 +25892,7 @@ interactions: uri: http://localhost:9200/_search/scroll response: body: - string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":6,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[]}}' + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn","took":10,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":11716,"relation":"eq"},"max_score":null,"hits":[]}}' headers: Transfer-Encoding: - chunked @@ -25904,7 +25904,7 @@ interactions: code: 200 message: OK - request: - body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFlJBcXdmTXZpVDlhVXJ2S2IyWEQwNGcAAAAAAFx0TRZWZ2lCN01PVFQzdW5KbFd6RHpmdERn"}' + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFjBxRGt1YnFsUVZLNzhVckx3MkJ0VXcAAAAAAAB3SxZWZ2lCN01PVFQzdW5KbFd6RHpmdERn"}' headers: accept: - application/vnd.elasticsearch+json; compatible-with=8 diff --git a/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_log_repo.py b/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_log_repo.py index 89c2ae9..f5bd921 100644 --- a/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_log_repo.py +++ b/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_log_repo.py @@ -1,11 +1,12 @@ import pytest +from dateutil import parser from logtools.log.sources.input.elastic_search.elastic_search_log_repo import ElasticSearchLogRepo, Namespace, Pod # XXX these are not good quality tests as they are overly complex and either tightly coupled to specific data or very -# weak in terms of what they assert. Ideally we should build simpler fixtures and test smaller bits at a time, but -# that requires a lot of setup, so we go with this. +# weak in terms of what they assert. They will be a pain to maintain. Ideally we should build simpler fixtures and +# test smaller bits at a time, but that requires a lot of setup, so for now we go with this. @pytest.mark.vcr def test_should_retrieve_existing_namespaces(): @@ -22,11 +23,15 @@ def test_should_retrieve_existing_namespaces(): '20231109-043100', '20231109-055106', '20231109-085853', + '20231114-045924', + '20231114-051016', + '20231114-051742', ), indices=( 'continuous-tests-pods-2023.11.07', 'continuous-tests-pods-2023.11.09', 'continuous-tests-pods-2023.11.10', + 'continuous-tests-pods-2023.11.14', ), ), Namespace( @@ -60,3 +65,16 @@ def test_should_retrieve_existing_pods_for_namespace(): 'continuous-tests-pods-2023.11.10', ) ) in pods + + +@pytest.mark.vcr +def test_should_respect_time_horizon_for_retrieving_resources(): + repo = ElasticSearchLogRepo(since=parser.parse('2023-11-14T18:00:00.000Z')) + namespaces = repo.namespaces('codex-continuous-tests-profiling') + + assert len(list(namespaces)) == 0 + + repo = ElasticSearchLogRepo(since=parser.parse('2023-11-07T18:00:00.000Z')) + namespaces = repo.namespaces('codex-continuous-tests-profiling') + + assert len(list(namespaces)) == 2 diff --git a/logtools/log/sources/input/tests/test_file_source.py b/logtools/log/sources/input/tests/test_file_source.py index d9018d0..7ee501f 100644 --- a/logtools/log/sources/input/tests/test_file_source.py +++ b/logtools/log/sources/input/tests/test_file_source.py @@ -1,7 +1,7 @@ from pathlib import Path -from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource from logtools.log.sources.input.file_log_source import FileLogSource +from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource SAMPLE_LOG = Path(__file__).parent / 'sample.log' diff --git a/logtools/log/sources/parse/tests/test_chronicles_raw_source.py b/logtools/log/sources/parse/tests/test_chronicles_raw_source.py index 21ae4b4..b1d5ed2 100644 --- a/logtools/log/sources/parse/tests/test_chronicles_raw_source.py +++ b/logtools/log/sources/parse/tests/test_chronicles_raw_source.py @@ -3,8 +3,8 @@ from datetime import datetime import pytest import pytz -from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource, ChroniclesLogLine, LogLevel from logtools.log.sources.input.string_log_source import StringLogSource +from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource, ChroniclesLogLine, LogLevel def test_should_parse_raw_chronicles_logs():